howler-client 3.1.0.dev404__tar.gz → 3.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. howler_client-3.2.0/PKG-INFO +293 -0
  2. howler_client-3.2.0/README.md +260 -0
  3. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/__init__.py +11 -0
  4. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/client.py +2 -0
  5. howler_client-3.2.0/howler_client/common/exceptions.py +2 -0
  6. howler_client-3.2.0/howler_client/module/mapping.py +313 -0
  7. howler_client-3.2.0/howler_client/schemas/__init__.py +1 -0
  8. howler_client-3.2.0/howler_client/schemas/row_mapping_v1.schema.json +130 -0
  9. howler_client-3.2.0/howler_client/utils/transforms.py +135 -0
  10. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/pyproject.toml +7 -2
  11. howler_client-3.1.0.dev404/PKG-INFO +0 -73
  12. howler_client-3.1.0.dev404/README.md +0 -42
  13. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/LICENSE +0 -0
  14. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/__init__.py +0 -0
  15. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/dict_utils.py +0 -0
  16. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/utils.py +0 -0
  17. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/connection.py +0 -0
  18. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/logger.py +0 -0
  19. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/__init__.py +0 -0
  20. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/action.py +0 -0
  21. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/comment.py +0 -0
  22. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/help.py +0 -0
  23. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/hit.py +0 -0
  24. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/__init__.py +0 -0
  25. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/chunk.py +0 -0
  26. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/facet.py +0 -0
  27. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/fields.py +0 -0
  28. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/grouped.py +0 -0
  29. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/histogram.py +0 -0
  30. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/stats.py +0 -0
  31. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/stream.py +0 -0
  32. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/user.py +0 -0
  33. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/__init__.py +0 -0
  34. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/case.py +0 -0
  35. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/ingest.py +0 -0
  36. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/search.py +0 -0
  37. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/utils/__init__.py +0 -0
  38. {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/utils/json_encoders.py +0 -0
@@ -0,0 +1,293 @@
1
+ Metadata-Version: 2.4
2
+ Name: howler-client
3
+ Version: 3.2.0
4
+ Summary: The Howler client library facilitates issuing requests to Howler
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
8
+ Author: Canadian Centre for Cyber Security
9
+ Author-email: howler@cyber.gc.ca
10
+ Requires-Python: >=3.10,<4.0
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Programming Language :: Python :: 3.15
21
+ Classifier: Topic :: Software Development :: Libraries
22
+ Requires-Dist: coverage[toml] (>=7.6.1,<8.0.0)
23
+ Requires-Dist: jsonschema (>=4.23,<5.0)
24
+ Requires-Dist: pycryptodome (>=3.20.0,<4.0.0)
25
+ Requires-Dist: python-baseconv (>=1.2.2,<2.0.0)
26
+ Requires-Dist: pyyaml (>=6.0,<7)
27
+ Requires-Dist: requests[security] (>=2.32.0,<3.0.0)
28
+ Project-URL: Documentation, https://cybercentrecanada.github.io/howler/developer/client/
29
+ Project-URL: Homepage, https://cybercentrecanada.github.io/howler/
30
+ Project-URL: Repository, https://github.com/CybercentreCanada/howler-client
31
+ Description-Content-Type: text/markdown
32
+
33
+ # Howler Client Library
34
+
35
+ `howler-client` is the Python client for the Howler API. Use it to:
36
+
37
+ - create, search, update, and delete hits
38
+ - manage comments, actions, users, and cases
39
+ - turn raw table rows into hit records offline, before you ingest them
40
+
41
+ It handles authentication, request formatting, retries, and response parsing.
42
+
43
+ - **Package:** [howler-client on PyPI](https://pypi.org/project/howler-client/)
44
+ - **User guide:** [Howler client documentation](https://cybercentrecanada.github.io/howler/ingestion/client/)
45
+ - **Contributor guide:** [Client development](https://cybercentrecanada.github.io/howler/developer/client/)
46
+
47
+ ## Installation
48
+
49
+ The client requires Python 3.10 or newer.
50
+
51
+ ```bash
52
+ pip install howler-client
53
+ ```
54
+
55
+ ## Connecting to Howler
56
+
57
+ `get_client()` returns a `Client` connected to a Howler server. It checks that the server supports the v1 API when
58
+ it connects, so the server must be reachable.
59
+
60
+ ```python
61
+ from howler_client import get_client
62
+
63
+ # API key (recommended): create one in the Howler UI under Settings -> User Security -> API Keys
64
+ client = get_client("https://howler.example.com", apikey=("my_username", "my_key_name:my_key_secret"))
65
+
66
+ # JWT, for example one obtained through an OAuth flow
67
+ client = get_client("https://howler.example.com", auth="eyJhbGciOi...")
68
+
69
+ # Username and password (only if the server allows internal authentication)
70
+ client = get_client("https://howler.example.com", auth=("my_username", "my_password"))
71
+ ```
72
+
73
+ Other useful `get_client()` options:
74
+
75
+ - `verify`: whether to verify TLS certificates.
76
+ - `cert`: a client certificate.
77
+ - `timeout`: the request timeout.
78
+ - `retries`: how many times to retry a request. The default `0` retries forever.
79
+ - `throw_on_bad_request` and `throw_on_max_retries`: whether to raise `ClientError` on failures.
80
+ - `authenticate`: a callback that returns a bearer token.
81
+
82
+ ## What the client covers
83
+
84
+ | Attribute | Purpose |
85
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
86
+ | `client.hit` | Create hits (`create`), fetch one by ID (`client.hit(hit_id)`), `overwrite`, `update`, `update_by_query`, `delete`, `generate_hash`. Comments are under `client.hit.comment` (`add`, `edit`, `delete`). |
87
+ | `client.search` | Lucene search over hits (`hit`) and actions (`action`), plus `facet`, `fields`, `grouped`, `histogram`, `stats`, and `stream` helpers, each with a `.hit(...)` method. |
88
+ | `client.action` | List, create, and execute saved actions. |
89
+ | `client.user` | Add, delete, list, and update users; `whoami`. |
90
+ | `client.help` | Server metadata such as `classification_definition()`. |
91
+ | `client.v2.ingest` | v2 ingestion for the `hit` and `event` indexes: `create`, `validate` (check records without storing them), `overwrite`, `update_by_query`, `delete`. |
92
+ | `client.v2.search` | Search across several indexes at once (`client.v2.search("hit,event", query)`), plus `count`, `facet`, and `explain`. |
93
+ | `client.v2.case` | Fetch, create, update, hide, and delete cases; manage case items and correlation rules. |
94
+ | `client.mapping` | `map_rows(...)`, the same as the top-level `howler_client.map_rows`. See below. |
95
+
96
+ For example:
97
+
98
+ ```python
99
+ results = client.search.hit("howler.status:open", rows=10, sort="event.created desc")
100
+ for hit in results["items"]:
101
+ print(hit["howler"]["id"], hit["howler"]["analytic"])
102
+
103
+ client.hit.create({"howler": {"analytic": "My Analytic", "score": 0.5}})
104
+ ```
105
+
106
+ Failed requests raise `howler_client.common.utils.ClientError`. It has `status_code` and `api_response` attributes. See
107
+ the [user guide](https://cybercentrecanada.github.io/howler/ingestion/client/) for more examples.
108
+
109
+ ## Mapping raw table rows to hit records
110
+
111
+ `RowMapping` turns raw rows into Howler hit dictionaries using a YAML mapping file. It runs entirely offline:
112
+
113
+ - It does not need a Howler connection.
114
+ - It never stores or ingests records.
115
+ - It never sets server-owned IDs such as `howler.id`.
116
+
117
+ To store the results, pass them to `client.hit.create` or `client.v2.ingest.create`.
118
+
119
+ ```python
120
+ from howler_client import RowMapping, map_rows
121
+
122
+ mapping = RowMapping.from_file("mapping.yaml")
123
+
124
+ for hit, error in map_rows(query_result_rows, mapping):
125
+ if error is not None:
126
+ handle_mapping_error(error) # a MappingError naming the row number and target field
127
+ continue
128
+ process_hit(hit)
129
+ ```
130
+
131
+ How `map_rows` behaves:
132
+
133
+ - It is lazy: it reads each source row only when you ask for the next result.
134
+ - For each row, in order, it yields either `(hit, None)` or `(None, MappingError)`.
135
+ - A bad row produces an error result, and the following rows are still processed.
136
+ - A bad mapping raises as soon as you call `map_rows`, before any row is read.
137
+ - Exceptions raised by your row iterator are passed through unchanged.
138
+
139
+ To map a single row, `mapping.map_row(row)` returns the hit or raises `MappingError`.
140
+
141
+ ### Mapping file format
142
+
143
+ ```yaml
144
+ version: 1
145
+ on_null: error # default for every target: error (default) or omit
146
+ mappings:
147
+ howler.analytic:
148
+ value: Directory User Inventory # constant
149
+ user.id:
150
+ column: user_identifier # copy a column
151
+ user.name:
152
+ column: account_name
153
+ transforms: [trim, lower]
154
+ user.email:
155
+ column: primary_email
156
+ on_null: omit # leave the field out when the value is null
157
+ organization.name:
158
+ column: profile
159
+ path: [organization, name] # read a value nested inside the column
160
+ user.roles:
161
+ concat: [directory_role_names, elevated_role_names] # join list columns
162
+ ```
163
+
164
+ Each key under `mappings` is a dotted Howler field name. Each target needs exactly one source:
165
+
166
+ | Expression | Result |
167
+ | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
168
+ | `column: name` | The value of that column. A missing column is a row error. |
169
+ | `column` + `path: [key, 0, ...]` | A value nested inside the column, found by dictionary keys and list indexes. A missing nested key or index counts as null. |
170
+ | `concat: [a, b, ...]` | Two or more list columns joined into one list. |
171
+ | `value: ...` | A constant, computed once when the mapping is loaded. |
172
+
173
+ A target can also set:
174
+
175
+ - `on_null`, which overrides the document-wide setting. With `error`, a null value fails the row. With `omit`, the
176
+ field is left out.
177
+ - `transforms`, which are applied in order:
178
+
179
+ | Transform | Effect |
180
+ | ------------------------------ | ------------------------------------------------------------------------------------------------- |
181
+ | `string` | Converts a scalar to a string. Use `json` for objects and lists. |
182
+ | `integer`, `number`, `boolean` | Strict conversions. `boolean` accepts only booleans or `"true"`/`"false"`. |
183
+ | `trim`, `lower`, `upper` | String clean-up. |
184
+ | `list` | Wraps a scalar in a list. |
185
+ | `json` | Converts the value to compact JSON text with sorted keys. |
186
+ | `{op: timestamp}` | Converts an ISO-8601 string to UTC. Add `assume_timezone: UTC` to accept values with no timezone. |
187
+
188
+ Mapping files are parsed with `yaml.safe_load`. Quote constants that must stay strings, such as `"yes"`, `"off"`, or
189
+ `"2026-01-01"`.
190
+
191
+ Mapping documents must contain JSON-compatible values: object keys must be strings, and YAML-only values such as dates,
192
+ sets, and binary data (as well as non-finite numbers and cyclic structures) are rejected. The client validates mappings
193
+ offline with its bundled [Draft 2020-12 schema](howler_client/schemas/row_mapping_v1.schema.json), which is included in
194
+ the wheel and source distribution.
195
+
196
+ You can also build a mapping from YAML text with `RowMapping.from_yaml(text)`, or from an already-loaded dictionary
197
+ with `RowMapping(document)`. `map_rows` accepts a `RowMapping`, a dictionary, or a path to a YAML file. To use YAML
198
+ text, go through `from_yaml` so it isn't mistaken for a path.
199
+ [`examples/row_mapping/example_mapping.yaml`](examples/row_mapping/example_mapping.yaml) is a complete example.
200
+
201
+ ### Checking targets against Howler's fields
202
+
203
+ You can pass Howler's field metadata to check targets before you ingest anything. Use the output of
204
+ `client.search.fields.hit()` or of the MCP `get_hit_fields` tool:
205
+
206
+ ```python
207
+ fields = client.search.fields.hit()
208
+ mapping = RowMapping.from_file("mapping.yaml", fields=fields)
209
+ ```
210
+
211
+ With field metadata, `RowMapping` checks that:
212
+
213
+ - every target exists and is not deprecated
214
+ - every `concat` target is a list field
215
+
216
+ `RowMapping` does not validate mapped values against target field types or list shape. Validate records with
217
+ `client.v2.ingest.validate("hit", hits)` or let Howler validate them during ingestion.
218
+
219
+ ## Development
220
+
221
+ The client lives in the `client/` folder of the Howler monorepo. It uses its own [Poetry](https://python-poetry.org/)
222
+ environment, so run every command below from `client/`.
223
+
224
+ ### Setup
225
+
226
+ ```bash
227
+ cd client
228
+ poetry install --with dev,test,types
229
+ ```
230
+
231
+ The repository's pre-commit hooks run Ruff and check commit messages. From the repository root, install them with
232
+ `pre-commit install`.
233
+
234
+ ### Code quality
235
+
236
+ These are the checks CI runs:
237
+
238
+ ```bash
239
+ poetry run ruff format howler_client --diff
240
+ poetry run ruff check howler_client --output-format=github
241
+ poetry run type_check # mypy over howler_client
242
+ ```
243
+
244
+ ### Unit tests
245
+
246
+ Unit tests in `test/unit/` run offline: they need no Howler server, Elasticsearch, or Redis.
247
+
248
+ ```bash
249
+ poetry run test test/unit # all unit tests
250
+ poetry run test test/unit/test_row_mapping.py -k timestamp # focused run; arguments are passed to pytest
251
+ ```
252
+
253
+ ### Integration tests
254
+
255
+ Integration tests in `test/integration/` call a running Howler API at `http://localhost:5000`. Set `UI_HOST` to use a
256
+ different address. The API needs Elasticsearch, Redis, and seeded test data. To set this up the same way CI does,
257
+ starting from the repository root:
258
+
259
+ ```bash
260
+ # 1. Prepare the API configuration
261
+ sudo mkdir -p /etc/howler/conf /etc/howler/lookups /var/log/howler
262
+ sudo chown -R "$USER" /etc/howler /var/log/howler
263
+ cp api/build_scripts/classification.yml /etc/howler/conf/classification.yml
264
+ cp api/build_scripts/mappings.yml /etc/howler/conf/mappings.yml
265
+ cp client/test/config/config.ci.yml /etc/howler/conf/config.yml
266
+
267
+ # 2. Start Elasticsearch and Redis
268
+ docker compose -f client/test/docker-compose.yml up -d
269
+
270
+ # 3. Install and start the API (leave it running in its own terminal)
271
+ cd api
272
+ poetry install --with test,dev
273
+ poetry run mitre /etc/howler/lookups
274
+ poetry run sigma
275
+ TESTING=true poetry run flask --app howler.app run --no-reload
276
+
277
+ # 4. In another terminal, seed users and random data (this creates the admin API key the tests use)
278
+ cd api
279
+ poetry run python -m howler.odm.random_data
280
+
281
+ # 5. Run the full client test suite with coverage
282
+ cd ../client
283
+ poetry run test
284
+ ```
285
+
286
+ `poetry run test` with no arguments runs everything under `test/` with branch coverage. Integration tests need the
287
+ running services. To run only part of the suite, pass paths or pytest options.
288
+
289
+ ### Releases
290
+
291
+ Releases are automated with Release Please using Conventional Commits. Use the `client` scope, for example
292
+ `feat(client): ...`. `CHANGELOG.md` is generated, so don't edit it by hand.
293
+
@@ -0,0 +1,260 @@
1
+ # Howler Client Library
2
+
3
+ `howler-client` is the Python client for the Howler API. Use it to:
4
+
5
+ - create, search, update, and delete hits
6
+ - manage comments, actions, users, and cases
7
+ - turn raw table rows into hit records offline, before you ingest them
8
+
9
+ It handles authentication, request formatting, retries, and response parsing.
10
+
11
+ - **Package:** [howler-client on PyPI](https://pypi.org/project/howler-client/)
12
+ - **User guide:** [Howler client documentation](https://cybercentrecanada.github.io/howler/ingestion/client/)
13
+ - **Contributor guide:** [Client development](https://cybercentrecanada.github.io/howler/developer/client/)
14
+
15
+ ## Installation
16
+
17
+ The client requires Python 3.10 or newer.
18
+
19
+ ```bash
20
+ pip install howler-client
21
+ ```
22
+
23
+ ## Connecting to Howler
24
+
25
+ `get_client()` returns a `Client` connected to a Howler server. It checks that the server supports the v1 API when
26
+ it connects, so the server must be reachable.
27
+
28
+ ```python
29
+ from howler_client import get_client
30
+
31
+ # API key (recommended): create one in the Howler UI under Settings -> User Security -> API Keys
32
+ client = get_client("https://howler.example.com", apikey=("my_username", "my_key_name:my_key_secret"))
33
+
34
+ # JWT, for example one obtained through an OAuth flow
35
+ client = get_client("https://howler.example.com", auth="eyJhbGciOi...")
36
+
37
+ # Username and password (only if the server allows internal authentication)
38
+ client = get_client("https://howler.example.com", auth=("my_username", "my_password"))
39
+ ```
40
+
41
+ Other useful `get_client()` options:
42
+
43
+ - `verify`: whether to verify TLS certificates.
44
+ - `cert`: a client certificate.
45
+ - `timeout`: the request timeout.
46
+ - `retries`: how many times to retry a request. The default `0` retries forever.
47
+ - `throw_on_bad_request` and `throw_on_max_retries`: whether to raise `ClientError` on failures.
48
+ - `authenticate`: a callback that returns a bearer token.
49
+
50
+ ## What the client covers
51
+
52
+ | Attribute | Purpose |
53
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
54
+ | `client.hit` | Create hits (`create`), fetch one by ID (`client.hit(hit_id)`), `overwrite`, `update`, `update_by_query`, `delete`, `generate_hash`. Comments are under `client.hit.comment` (`add`, `edit`, `delete`). |
55
+ | `client.search` | Lucene search over hits (`hit`) and actions (`action`), plus `facet`, `fields`, `grouped`, `histogram`, `stats`, and `stream` helpers, each with a `.hit(...)` method. |
56
+ | `client.action` | List, create, and execute saved actions. |
57
+ | `client.user` | Add, delete, list, and update users; `whoami`. |
58
+ | `client.help` | Server metadata such as `classification_definition()`. |
59
+ | `client.v2.ingest` | v2 ingestion for the `hit` and `event` indexes: `create`, `validate` (check records without storing them), `overwrite`, `update_by_query`, `delete`. |
60
+ | `client.v2.search` | Search across several indexes at once (`client.v2.search("hit,event", query)`), plus `count`, `facet`, and `explain`. |
61
+ | `client.v2.case` | Fetch, create, update, hide, and delete cases; manage case items and correlation rules. |
62
+ | `client.mapping` | `map_rows(...)`, the same as the top-level `howler_client.map_rows`. See below. |
63
+
64
+ For example:
65
+
66
+ ```python
67
+ results = client.search.hit("howler.status:open", rows=10, sort="event.created desc")
68
+ for hit in results["items"]:
69
+ print(hit["howler"]["id"], hit["howler"]["analytic"])
70
+
71
+ client.hit.create({"howler": {"analytic": "My Analytic", "score": 0.5}})
72
+ ```
73
+
74
+ Failed requests raise `howler_client.common.utils.ClientError`. It has `status_code` and `api_response` attributes. See
75
+ the [user guide](https://cybercentrecanada.github.io/howler/ingestion/client/) for more examples.
76
+
77
+ ## Mapping raw table rows to hit records
78
+
79
+ `RowMapping` turns raw rows into Howler hit dictionaries using a YAML mapping file. It runs entirely offline:
80
+
81
+ - It does not need a Howler connection.
82
+ - It never stores or ingests records.
83
+ - It never sets server-owned IDs such as `howler.id`.
84
+
85
+ To store the results, pass them to `client.hit.create` or `client.v2.ingest.create`.
86
+
87
+ ```python
88
+ from howler_client import RowMapping, map_rows
89
+
90
+ mapping = RowMapping.from_file("mapping.yaml")
91
+
92
+ for hit, error in map_rows(query_result_rows, mapping):
93
+ if error is not None:
94
+ handle_mapping_error(error) # a MappingError naming the row number and target field
95
+ continue
96
+ process_hit(hit)
97
+ ```
98
+
99
+ How `map_rows` behaves:
100
+
101
+ - It is lazy: it reads each source row only when you ask for the next result.
102
+ - For each row, in order, it yields either `(hit, None)` or `(None, MappingError)`.
103
+ - A bad row produces an error result, and the following rows are still processed.
104
+ - A bad mapping raises as soon as you call `map_rows`, before any row is read.
105
+ - Exceptions raised by your row iterator are passed through unchanged.
106
+
107
+ To map a single row, `mapping.map_row(row)` returns the hit or raises `MappingError`.
108
+
109
+ ### Mapping file format
110
+
111
+ ```yaml
112
+ version: 1
113
+ on_null: error # default for every target: error (default) or omit
114
+ mappings:
115
+ howler.analytic:
116
+ value: Directory User Inventory # constant
117
+ user.id:
118
+ column: user_identifier # copy a column
119
+ user.name:
120
+ column: account_name
121
+ transforms: [trim, lower]
122
+ user.email:
123
+ column: primary_email
124
+ on_null: omit # leave the field out when the value is null
125
+ organization.name:
126
+ column: profile
127
+ path: [organization, name] # read a value nested inside the column
128
+ user.roles:
129
+ concat: [directory_role_names, elevated_role_names] # join list columns
130
+ ```
131
+
132
+ Each key under `mappings` is a dotted Howler field name. Each target needs exactly one source:
133
+
134
+ | Expression | Result |
135
+ | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
136
+ | `column: name` | The value of that column. A missing column is a row error. |
137
+ | `column` + `path: [key, 0, ...]` | A value nested inside the column, found by dictionary keys and list indexes. A missing nested key or index counts as null. |
138
+ | `concat: [a, b, ...]` | Two or more list columns joined into one list. |
139
+ | `value: ...` | A constant, computed once when the mapping is loaded. |
140
+
141
+ A target can also set:
142
+
143
+ - `on_null`, which overrides the document-wide setting. With `error`, a null value fails the row. With `omit`, the
144
+ field is left out.
145
+ - `transforms`, which are applied in order:
146
+
147
+ | Transform | Effect |
148
+ | ------------------------------ | ------------------------------------------------------------------------------------------------- |
149
+ | `string` | Converts a scalar to a string. Use `json` for objects and lists. |
150
+ | `integer`, `number`, `boolean` | Strict conversions. `boolean` accepts only booleans or `"true"`/`"false"`. |
151
+ | `trim`, `lower`, `upper` | String clean-up. |
152
+ | `list` | Wraps a scalar in a list. |
153
+ | `json` | Converts the value to compact JSON text with sorted keys. |
154
+ | `{op: timestamp}` | Converts an ISO-8601 string to UTC. Add `assume_timezone: UTC` to accept values with no timezone. |
155
+
156
+ Mapping files are parsed with `yaml.safe_load`. Quote constants that must stay strings, such as `"yes"`, `"off"`, or
157
+ `"2026-01-01"`.
158
+
159
+ Mapping documents must contain JSON-compatible values: object keys must be strings, and YAML-only values such as dates,
160
+ sets, and binary data (as well as non-finite numbers and cyclic structures) are rejected. The client validates mappings
161
+ offline with its bundled [Draft 2020-12 schema](howler_client/schemas/row_mapping_v1.schema.json), which is included in
162
+ the wheel and source distribution.
163
+
164
+ You can also build a mapping from YAML text with `RowMapping.from_yaml(text)`, or from an already-loaded dictionary
165
+ with `RowMapping(document)`. `map_rows` accepts a `RowMapping`, a dictionary, or a path to a YAML file. To use YAML
166
+ text, go through `from_yaml` so it isn't mistaken for a path.
167
+ [`examples/row_mapping/example_mapping.yaml`](examples/row_mapping/example_mapping.yaml) is a complete example.
168
+
169
+ ### Checking targets against Howler's fields
170
+
171
+ You can pass Howler's field metadata to check targets before you ingest anything. Use the output of
172
+ `client.search.fields.hit()` or of the MCP `get_hit_fields` tool:
173
+
174
+ ```python
175
+ fields = client.search.fields.hit()
176
+ mapping = RowMapping.from_file("mapping.yaml", fields=fields)
177
+ ```
178
+
179
+ With field metadata, `RowMapping` checks that:
180
+
181
+ - every target exists and is not deprecated
182
+ - every `concat` target is a list field
183
+
184
+ `RowMapping` does not validate mapped values against target field types or list shape. Validate records with
185
+ `client.v2.ingest.validate("hit", hits)` or let Howler validate them during ingestion.
186
+
187
+ ## Development
188
+
189
+ The client lives in the `client/` folder of the Howler monorepo. It uses its own [Poetry](https://python-poetry.org/)
190
+ environment, so run every command below from `client/`.
191
+
192
+ ### Setup
193
+
194
+ ```bash
195
+ cd client
196
+ poetry install --with dev,test,types
197
+ ```
198
+
199
+ The repository's pre-commit hooks run Ruff and check commit messages. From the repository root, install them with
200
+ `pre-commit install`.
201
+
202
+ ### Code quality
203
+
204
+ These are the checks CI runs:
205
+
206
+ ```bash
207
+ poetry run ruff format howler_client --diff
208
+ poetry run ruff check howler_client --output-format=github
209
+ poetry run type_check # mypy over howler_client
210
+ ```
211
+
212
+ ### Unit tests
213
+
214
+ Unit tests in `test/unit/` run offline: they need no Howler server, Elasticsearch, or Redis.
215
+
216
+ ```bash
217
+ poetry run test test/unit # all unit tests
218
+ poetry run test test/unit/test_row_mapping.py -k timestamp # focused run; arguments are passed to pytest
219
+ ```
220
+
221
+ ### Integration tests
222
+
223
+ Integration tests in `test/integration/` call a running Howler API at `http://localhost:5000`. Set `UI_HOST` to use a
224
+ different address. The API needs Elasticsearch, Redis, and seeded test data. To set this up the same way CI does,
225
+ starting from the repository root:
226
+
227
+ ```bash
228
+ # 1. Prepare the API configuration
229
+ sudo mkdir -p /etc/howler/conf /etc/howler/lookups /var/log/howler
230
+ sudo chown -R "$USER" /etc/howler /var/log/howler
231
+ cp api/build_scripts/classification.yml /etc/howler/conf/classification.yml
232
+ cp api/build_scripts/mappings.yml /etc/howler/conf/mappings.yml
233
+ cp client/test/config/config.ci.yml /etc/howler/conf/config.yml
234
+
235
+ # 2. Start Elasticsearch and Redis
236
+ docker compose -f client/test/docker-compose.yml up -d
237
+
238
+ # 3. Install and start the API (leave it running in its own terminal)
239
+ cd api
240
+ poetry install --with test,dev
241
+ poetry run mitre /etc/howler/lookups
242
+ poetry run sigma
243
+ TESTING=true poetry run flask --app howler.app run --no-reload
244
+
245
+ # 4. In another terminal, seed users and random data (this creates the admin API key the tests use)
246
+ cd api
247
+ poetry run python -m howler.odm.random_data
248
+
249
+ # 5. Run the full client test suite with coverage
250
+ cd ../client
251
+ poetry run test
252
+ ```
253
+
254
+ `poetry run test` with no arguments runs everything under `test/` with branch coverage. Integration tests need the
255
+ running services. To run only part of the suite, pass paths or pytest options.
256
+
257
+ ### Releases
258
+
259
+ Releases are automated with Release Please using Conventional Commits. Use the `client` scope, for example
260
+ `feat(client): ...`. `CHANGELOG.md` is generated, so don't edit it by hand.
@@ -2,6 +2,7 @@ from importlib.metadata import PackageNotFoundError, version
2
2
 
3
3
  from howler_client.client import Client
4
4
  from howler_client.connection import Connection
5
+ from howler_client.module.mapping import MappingError, RowMapping, RowMappingResult, map_rows
5
6
 
6
7
  try:
7
8
  __version__ = version("howler-client")
@@ -11,6 +12,16 @@ except PackageNotFoundError:
11
12
  RETRY_FOREVER = 0
12
13
  SUPPORTED_APIS = {"v1"}
13
14
 
15
+ __all__ = [
16
+ "Client",
17
+ "Connection",
18
+ "MappingError",
19
+ "RowMapping",
20
+ "RowMappingResult",
21
+ "get_client",
22
+ "map_rows",
23
+ ]
24
+
14
25
 
15
26
  def get_client(
16
27
  server,
@@ -5,6 +5,7 @@ from howler_client.connection import Connection
5
5
  from howler_client.module.action import Action
6
6
  from howler_client.module.help import Help
7
7
  from howler_client.module.hit import Hit
8
+ from howler_client.module.mapping import HowlerMapping
8
9
  from howler_client.module.search import Search
9
10
  from howler_client.module.user import User
10
11
  from howler_client.module.v2 import V2
@@ -27,6 +28,7 @@ class Client(object):
27
28
  self.hit = Hit(self._connection, self.search)
28
29
  self.user = User(self._connection)
29
30
  self.v2 = V2(self._connection)
31
+ self.mapping = HowlerMapping()
30
32
 
31
33
  paths: list[str] = []
32
34
  walk_api_path(self, [""], paths)
@@ -0,0 +1,2 @@
1
+ class MappingError(ValueError):
2
+ """A mapping document, field definition, or source row is invalid."""