datamasque-python 1.0.5__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,8 +17,10 @@ from datamasque.client.exceptions import (
17
17
  DataMasqueNotReadyError,
18
18
  DataMasqueTransportError,
19
19
  DataMasqueUserError,
20
+ DiscoveryConfigNotFoundError,
20
21
  FailedToStartError,
21
22
  IfmAuthError,
23
+ InvalidDiscoveryConfigError,
22
24
  InvalidLibraryError,
23
25
  InvalidRulesetError,
24
26
  RunNotCancellableError,
@@ -54,10 +56,15 @@ from datamasque.client.models.data_selection import (
54
56
  from datamasque.client.models.discovery import (
55
57
  ConstraintColumns,
56
58
  DiscoveryMatch,
59
+ FileDataDiscoveryFromConfigRequest,
60
+ FileDataDiscoveryOptions,
61
+ FileDataDiscoveryRequest,
57
62
  FileDiscoveryFile,
58
63
  FileDiscoveryLocatorResult,
59
64
  FileDiscoveryMatch,
60
65
  FileDiscoveryResult,
66
+ FileFilter,
67
+ FileFilterMatchAgainst,
61
68
  FileRulesetGenerationRequest,
62
69
  ForeignKeyRef,
63
70
  InDataDiscoveryConfig,
@@ -65,11 +72,13 @@ from datamasque.client.models.discovery import (
65
72
  ReferencingForeignKey,
66
73
  RulesetGenerationRequest,
67
74
  SchemaDiscoveryColumn,
75
+ SchemaDiscoveryFromConfigRequest,
68
76
  SchemaDiscoveryPage,
69
77
  SchemaDiscoveryRequest,
70
78
  SchemaDiscoveryResult,
71
79
  TableConstraints,
72
80
  )
81
+ from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
73
82
  from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
74
83
  from datamasque.client.models.files import (
75
84
  DataMasqueFile,
@@ -79,6 +88,7 @@ from datamasque.client.models.files import (
79
88
  SnowflakeKeyFile,
80
89
  SslZipFile,
81
90
  )
91
+ from datamasque.client.models.git import GitSnapshot
82
92
  from datamasque.client.models.ifm import (
83
93
  DataMasqueIfmInstanceConfig,
84
94
  IfmLog,
@@ -104,7 +114,12 @@ from datamasque.client.models.runs import (
104
114
  RunInfo,
105
115
  UnfinishedRun,
106
116
  )
107
- from datamasque.client.models.status import AsyncRulesetGenerationTaskStatus, MaskingRunStatus, ValidationStatus
117
+ from datamasque.client.models.status import (
118
+ AsyncRulesetGenerationTaskStatus,
119
+ MaskingRunStatus,
120
+ ValidationErrorType,
121
+ ValidationStatus,
122
+ )
108
123
  from datamasque.client.models.user import User, UserId, UserRole
109
124
 
110
125
  __version__ = version("datamasque-python")
@@ -130,18 +145,28 @@ __all__ = [
130
145
  "DatabaseConnectionConfig",
131
146
  "DatabaseType",
132
147
  "DatabricksConnectionConfig",
148
+ "DiscoveryConfig",
149
+ "DiscoveryConfigId",
150
+ "DiscoveryConfigNotFoundError",
151
+ "DiscoveryConfigType",
133
152
  "DiscoveryMatch",
134
153
  "DynamoConnectionConfig",
135
154
  "FailedToStartError",
136
155
  "FileConnectionConfig",
156
+ "FileDataDiscoveryFromConfigRequest",
157
+ "FileDataDiscoveryOptions",
158
+ "FileDataDiscoveryRequest",
137
159
  "FileDiscoveryFile",
138
160
  "FileDiscoveryLocatorResult",
139
161
  "FileDiscoveryMatch",
140
162
  "FileDiscoveryResult",
163
+ "FileFilter",
164
+ "FileFilterMatchAgainst",
141
165
  "FileId",
142
166
  "FileOrContent",
143
167
  "FileRulesetGenerationRequest",
144
168
  "ForeignKeyRef",
169
+ "GitSnapshot",
145
170
  "HashColumnsTableConfig",
146
171
  "IfmAuthError",
147
172
  "IfmLog",
@@ -151,6 +176,7 @@ __all__ = [
151
176
  "IfmTokenInfo",
152
177
  "InDataDiscoveryConfig",
153
178
  "InDataDiscoveryRule",
179
+ "InvalidDiscoveryConfigError",
154
180
  "InvalidLibraryError",
155
181
  "InvalidRulesetError",
156
182
  "JsonPath",
@@ -182,6 +208,7 @@ __all__ = [
182
208
  "RunNotCancellableError",
183
209
  "S3ConnectionConfig",
184
210
  "SchemaDiscoveryColumn",
211
+ "SchemaDiscoveryFromConfigRequest",
185
212
  "SchemaDiscoveryPage",
186
213
  "SchemaDiscoveryRequest",
187
214
  "SchemaDiscoveryResult",
@@ -202,5 +229,6 @@ __all__ = [
202
229
  "UserId",
203
230
  "UserRole",
204
231
  "UserSelection",
232
+ "ValidationErrorType",
205
233
  "ValidationStatus",
206
234
  ]
@@ -4,11 +4,15 @@ from io import BufferedIOBase, BytesIO, TextIOBase
4
4
  from pathlib import Path
5
5
  from typing import Iterator, Optional, Union
6
6
 
7
+ from requests import Response
8
+
7
9
  from datamasque.client.base import BaseClient, UploadFile
8
10
  from datamasque.client.exceptions import (
9
11
  AsyncRulesetGenerationInProgressError,
10
12
  DataMasqueException,
13
+ DiscoveryConfigNotFoundError,
11
14
  FailedToStartError,
15
+ InvalidDiscoveryConfigError,
12
16
  )
13
17
  from datamasque.client.models.connection import ConnectionId
14
18
  from datamasque.client.models.data_selection import (
@@ -17,9 +21,12 @@ from datamasque.client.models.data_selection import (
17
21
  SelectedFileData,
18
22
  )
19
23
  from datamasque.client.models.discovery import (
24
+ FileDataDiscoveryFromConfigRequest,
25
+ FileDataDiscoveryRequest,
20
26
  FileDiscoveryResult,
21
27
  FileRulesetGenerationRequest,
22
28
  RulesetGenerationRequest,
29
+ SchemaDiscoveryFromConfigRequest,
23
30
  SchemaDiscoveryPage,
24
31
  SchemaDiscoveryRequest,
25
32
  SchemaDiscoveryResult,
@@ -185,6 +192,13 @@ class DiscoveryClient(BaseClient):
185
192
  with zip_file.open(file_info) as file:
186
193
  yaml_content = file.read().decode("utf-8")
187
194
  rulesets.append(Ruleset(name=Path(file_info.filename).stem, yaml=yaml_content))
195
+
196
+ if not rulesets:
197
+ raise DataMasqueException(
198
+ f"Ruleset generation for connection {connection_id} reported `finished` "
199
+ f"but the downloaded archive contained no rulesets."
200
+ )
201
+
188
202
  return rulesets
189
203
 
190
204
  generated = response.json().get("generated_ruleset")
@@ -230,6 +244,148 @@ class DiscoveryClient(BaseClient):
230
244
  response=response,
231
245
  )
232
246
 
247
+ def start_file_data_discovery_run(self, request: FileDataDiscoveryRequest) -> RunId:
248
+ """
249
+ Starts a file data discovery run with the given configuration.
250
+
251
+ Args:
252
+ request: A `FileDataDiscoveryRequest` with connection and optional settings.
253
+
254
+ Returns:
255
+ RunId: The ID of the started discovery run
256
+
257
+ Raises:
258
+ FailedToStartError: If run fails to start
259
+ """
260
+
261
+ data = request.model_dump(exclude_none=True, mode="json")
262
+ response = self.make_request(
263
+ "POST",
264
+ "/api/run-file-data-discovery/",
265
+ data=data,
266
+ require_status_check=False,
267
+ )
268
+ run_data = response.json()
269
+
270
+ if response.status_code == 201:
271
+ logger.info("File data discovery run %s started successfully", run_data["id"])
272
+ return RunId(run_data["id"])
273
+
274
+ logger.error("File data discovery run failed to start: %s", run_data)
275
+ raise FailedToStartError(
276
+ f"File data discovery run failed to start "
277
+ f"(server responded with status {response.status_code}: {response.text}).",
278
+ response=response,
279
+ )
280
+
281
+ def start_schema_discovery_run_from_config(self, request: SchemaDiscoveryFromConfigRequest) -> RunId:
282
+ """
283
+ Starts a schema discovery run from a saved discovery config.
284
+
285
+ Args:
286
+ request: A `SchemaDiscoveryFromConfigRequest` with the `connection` and a required `discovery_config`
287
+ (a saved config, or `None` for the server's defaults).
288
+
289
+ Returns:
290
+ RunId: The ID of the started discovery run
291
+
292
+ Raises:
293
+ DiscoveryConfigNotFoundError: the referenced discovery config cannot be found
294
+ (it does not exist or is the wrong type for the run).
295
+ InvalidDiscoveryConfigError: the config is present but not in a `valid` validation state,
296
+ or its YAML is rejected when the run starts.
297
+ FailedToStartError: the run failed to start for any other reason.
298
+ """
299
+
300
+ return self._start_discovery_run_from_config(request, "/api/schema-discovery/v2/", "Schema discovery")
301
+
302
+ def start_file_data_discovery_run_from_config(self, request: FileDataDiscoveryFromConfigRequest) -> RunId:
303
+ """
304
+ Starts a file data discovery run from a saved discovery config.
305
+
306
+ Args:
307
+ request: A `FileDataDiscoveryFromConfigRequest` with the `connection`,
308
+ a required `discovery_config` (a saved config, or `None` for the server's defaults),
309
+ and optional run `options`.
310
+
311
+ Returns:
312
+ RunId: The ID of the started discovery run
313
+
314
+ Raises:
315
+ DiscoveryConfigNotFoundError: the referenced discovery config cannot be found
316
+ (it does not exist or is the wrong type for the run).
317
+ InvalidDiscoveryConfigError: the config is present but not in a `valid` validation state,
318
+ or its YAML is rejected when the run starts.
319
+ FailedToStartError: the run failed to start for any other reason.
320
+ """
321
+
322
+ return self._start_discovery_run_from_config(request, "/api/run-file-data-discovery/v2/", "File data discovery")
323
+
324
+ def _start_discovery_run_from_config(
325
+ self,
326
+ request: Union[SchemaDiscoveryFromConfigRequest, FileDataDiscoveryFromConfigRequest],
327
+ path: str,
328
+ run_kind: str,
329
+ ) -> RunId:
330
+ """Post a saved-config discovery request and return its run id, classifying config errors on failure."""
331
+
332
+ data = request.model_dump(exclude_none=True, mode="json")
333
+ # The server requires `discovery_config` to be present; a null selects its built-in defaults,
334
+ # so send it explicitly rather than letting `exclude_none` drop a None.
335
+ data.setdefault("discovery_config", None)
336
+ response = self.make_request("POST", path, data=data, require_status_check=False)
337
+ run_data = response.json() if response.content else {}
338
+
339
+ if response.status_code == 201:
340
+ logger.info("%s run %s started successfully", run_kind, run_data["id"])
341
+ return RunId(run_data["id"])
342
+
343
+ logger.error("%s run failed to start: %s", run_kind, run_data)
344
+ self._maybe_raise_discovery_config_error(run_data, response, run_kind)
345
+ raise FailedToStartError(
346
+ f"{run_kind} run failed to start (server responded with status {response.status_code}: {response.text}).",
347
+ response=response,
348
+ )
349
+
350
+ # Server key for a 400 that means the discovery config itself is unusable:
351
+ # a missing or wrong-type config, or one not in a `valid` validation state (string messages),
352
+ # or re-validation of broken saved-config YAML when the run starts
353
+ # (a `{"message", "line_number", "column_number"}` dict).
354
+ DISCOVERY_CONFIG_ERROR_FIELD = "discovery_config"
355
+
356
+ # The phrase the server uses when the config id cannot be resolved (a missing or wrong-type config).
357
+ MISSING_DISCOVERY_CONFIG_SIGNATURE = "object does not exist"
358
+
359
+ @classmethod
360
+ def _maybe_raise_discovery_config_error(cls, run_data: object, response: Response, run_kind: str) -> None:
361
+ """Raise a discovery-config error if the server's 400 body cites the discovery config."""
362
+ if not isinstance(run_data, dict):
363
+ return
364
+
365
+ if not (errors := run_data.get(cls.DISCOVERY_CONFIG_ERROR_FIELD)):
366
+ return
367
+
368
+ detail = cls._format_discovery_config_error(errors)
369
+ if cls.MISSING_DISCOVERY_CONFIG_SIGNATURE in detail:
370
+ raise DiscoveryConfigNotFoundError(
371
+ f"{run_kind} run failed to start: the referenced discovery config could not be found: {detail}",
372
+ response=response,
373
+ )
374
+
375
+ raise InvalidDiscoveryConfigError(
376
+ f"{run_kind} run failed to start due to discovery config error: {detail}",
377
+ response=response,
378
+ )
379
+
380
+ @staticmethod
381
+ def _format_discovery_config_error(errors: object) -> str:
382
+ """Render the first server error, handling both string and `{message, ...}` dict items."""
383
+ first = errors[0] if isinstance(errors, list) and errors else errors
384
+ if isinstance(first, dict) and "message" in first:
385
+ return str(first["message"])
386
+
387
+ return str(first)
388
+
233
389
  def iter_schema_discovery_results(self, run_id: RunId) -> Iterator[SchemaDiscoveryResult]:
234
390
  """Lazily iterate all schema discovery results for a run via the paginated v2 endpoint."""
235
391
 
@@ -284,3 +440,22 @@ class DiscoveryClient(BaseClient):
284
440
 
285
441
  response = self.make_request("GET", f"api/runs/{run_id}/file-discovery-results/")
286
442
  return [FileDiscoveryResult.model_validate(d) for d in response.json()]
443
+
444
+ def get_discovery_run_config_snapshot_yaml(self, run_id: RunId, *, timezone: Optional[str] = None) -> str:
445
+ """
446
+ Returns the discovery-config YAML that was effective at the start of the given discovery run.
447
+
448
+ The YAML is prefixed with a commented provenance header naming the saved config
449
+ (or the built-in defaults) the run used, and whether it has since been modified or deleted.
450
+ `timezone`, a `±HH:MM` UTC offset, sets the timezone of the header timestamp; the server defaults to UTC.
451
+ """
452
+
453
+ params = {"timezone": timezone} if timezone is not None else None
454
+ response = self.make_request("GET", f"/api/discovery/runs/{run_id}/config-snapshot/", params=params)
455
+ with zipfile.ZipFile(BytesIO(response.content)) as zip_file:
456
+ names = zip_file.namelist()
457
+ if not names:
458
+ raise DataMasqueException(f"Discovery run {run_id} config snapshot archive contained no files.")
459
+
460
+ with zip_file.open(names[0]) as snapshot_file:
461
+ return snapshot_file.read().decode("utf-8")
@@ -0,0 +1,158 @@
1
+ import logging
2
+ from typing import Iterator, Optional
3
+
4
+ from datamasque.client.base import BaseClient
5
+ from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException
6
+ from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
7
+ from datamasque.client.models.pagination import Page
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ class DiscoveryConfigClient(BaseClient):
13
+ """Discovery config CRUD API methods. Mixed into `DataMasqueClient`."""
14
+
15
+ def iter_discovery_configs(self) -> Iterator[DiscoveryConfig]:
16
+ """Lazily iterate all discovery configs via the paginated endpoint."""
17
+
18
+ return self._iter_paginated("/api/discovery/configs/", model=DiscoveryConfig)
19
+
20
+ def list_discovery_configs(self) -> list[DiscoveryConfig]:
21
+ """
22
+ Lists all discovery configs.
23
+
24
+ Note: the YAML content is not included in the list response for performance.
25
+ Use `get_discovery_config` to retrieve the full config with its YAML body.
26
+ """
27
+
28
+ return list(self.iter_discovery_configs())
29
+
30
+ def get_discovery_config(self, config_id: DiscoveryConfigId) -> DiscoveryConfig:
31
+ """Retrieves a single discovery config by ID."""
32
+
33
+ response = self.make_request("GET", f"/api/discovery/configs/{config_id}/")
34
+ return DiscoveryConfig.model_validate(response.json())
35
+
36
+ def _get_discovery_config_id_by_name(
37
+ self, name: str, config_type: DiscoveryConfigType
38
+ ) -> Optional[DiscoveryConfigId]:
39
+ """Return the id of the config matching name and type via a single list request, or `None`."""
40
+
41
+ response = self.make_request(
42
+ "GET",
43
+ "/api/discovery/configs/",
44
+ params={"name_exact": name, "config_type": config_type.value, "limit": 1},
45
+ )
46
+ page = Page[DiscoveryConfig].model_validate(response.json())
47
+ if not page.results:
48
+ return None
49
+
50
+ config_id = page.results[0].id
51
+ if config_id is None:
52
+ raise DataMasqueApiError(
53
+ "Server returned a discovery config list entry without an `id`.",
54
+ response=response,
55
+ )
56
+
57
+ return config_id
58
+
59
+ def get_discovery_config_by_name(self, name: str, config_type: DiscoveryConfigType) -> Optional[DiscoveryConfig]:
60
+ """
61
+ Looks for a discovery config matching the given name and type (case-sensitive, exact match).
62
+
63
+ Config names are unique per type, so a type is required to identify a single config.
64
+ Returns it if found, otherwise `None`.
65
+ """
66
+
67
+ config_id = self._get_discovery_config_id_by_name(name, config_type)
68
+ if config_id is None:
69
+ return None
70
+
71
+ return self.get_discovery_config(config_id)
72
+
73
+ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig:
74
+ """
75
+ Creates a new discovery config on the server.
76
+
77
+ Sets the config's server-assigned fields
78
+ (`id`, `is_valid`, `validation_error`, `created`, `modified`) and returns the config.
79
+ """
80
+
81
+ data = config.model_dump(exclude_none=True, by_alias=True, mode="json")
82
+ response = self.make_request("POST", "/api/discovery/configs/", data=data)
83
+ created = DiscoveryConfig.model_validate(response.json())
84
+ config.id = created.id
85
+ config.is_valid = created.is_valid
86
+ config.validation_error = created.validation_error
87
+ config.created = created.created
88
+ config.modified = created.modified
89
+ logger.info('Creation of discovery config "%s" successful', config.name)
90
+ return config
91
+
92
+ def update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig:
93
+ """
94
+ Performs a full update of the discovery config.
95
+
96
+ The config must have its `id` set
97
+ (i.e., it must have been previously created or retrieved from the server).
98
+ """
99
+
100
+ if config.id is None:
101
+ raise ValueError("Cannot update a discovery config that has not been created yet (id is None)")
102
+
103
+ data = config.model_dump(exclude_none=True, by_alias=True, mode="json")
104
+ response = self.make_request("PUT", f"/api/discovery/configs/{config.id}/", data=data)
105
+ updated = DiscoveryConfig.model_validate(response.json())
106
+ config.is_valid = updated.is_valid
107
+ config.validation_error = updated.validation_error
108
+ config.modified = updated.modified
109
+ logger.debug('Update of discovery config "%s" successful', config.name)
110
+ return config
111
+
112
+ def create_or_update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig:
113
+ """
114
+ Creates the config if it doesn't exist, or updates it if one with the same name already exists.
115
+
116
+ Sets the config's `id` property.
117
+ """
118
+
119
+ existing_id = self._get_discovery_config_id_by_name(config.name, config.config_type)
120
+ if existing_id is not None:
121
+ config.id = existing_id
122
+ return self.update_discovery_config(config)
123
+
124
+ return self.create_discovery_config(config)
125
+
126
+ def delete_discovery_config_by_id_if_exists(self, config_id: DiscoveryConfigId) -> None:
127
+ """
128
+ Deletes the discovery config with the given ID.
129
+
130
+ No-op if the config does not exist.
131
+ """
132
+
133
+ self._delete_if_exists(f"/api/discovery/configs/{config_id}/")
134
+
135
+ def delete_discovery_config_by_name_if_exists(self, name: str, config_type: DiscoveryConfigType) -> None:
136
+ """
137
+ Deletes the discovery config with the given name and type.
138
+
139
+ Config names are unique per type, so a type is required to identify a single config.
140
+ No-op if no such config exists.
141
+ """
142
+
143
+ matching = [
144
+ config
145
+ for config in self.list_discovery_configs()
146
+ if config.name == name and config.config_type is config_type
147
+ ]
148
+ for config in matching:
149
+ if config.id is None:
150
+ raise DataMasqueException(f'Server returned a discovery config named "{config.name}" without an `id`.')
151
+
152
+ self.delete_discovery_config_by_id_if_exists(config.id)
153
+
154
+ def get_default_discovery_config_yaml(self) -> str:
155
+ """Returns the server's built-in default discovery configuration as a YAML string."""
156
+
157
+ response = self.make_request("GET", "/api/discovery/configs/defaults/")
158
+ return response.content.decode("utf-8")
@@ -1,6 +1,7 @@
1
1
  from datamasque.client.base import FileOrContent, UploadFile
2
2
  from datamasque.client.connections import ConnectionClient
3
3
  from datamasque.client.discovery import DiscoveryClient
4
+ from datamasque.client.discovery_configs import DiscoveryConfigClient
4
5
  from datamasque.client.files import FileClient
5
6
  from datamasque.client.license import LicenseClient
6
7
  from datamasque.client.ruleset_libraries import RulesetLibraryClient
@@ -20,6 +21,7 @@ class DataMasqueClient(
20
21
  FileClient,
21
22
  RunClient,
22
23
  DiscoveryClient,
24
+ DiscoveryConfigClient,
23
25
  UserClient,
24
26
  SettingsClient,
25
27
  ):
@@ -41,6 +41,23 @@ class InvalidLibraryError(FailedToStartError):
41
41
  """Specific error for when runs fail to start due to having an invalid ruleset library."""
42
42
 
43
43
 
44
+ class InvalidDiscoveryConfigError(FailedToStartError):
45
+ """
46
+ Raised when a discovery run fails to start because the referenced config is present but unusable.
47
+
48
+ The config exists but is not in a `valid` validation state, or its YAML is rejected when the run starts.
49
+ A config that cannot be found raises `DiscoveryConfigNotFoundError` instead.
50
+ """
51
+
52
+
53
+ class DiscoveryConfigNotFoundError(FailedToStartError):
54
+ """
55
+ Raised when a discovery run references a discovery config that cannot be found.
56
+
57
+ The config does not exist or is the wrong type for the run.
58
+ """
59
+
60
+
44
61
  class DataMasqueTransportError(DataMasqueException):
45
62
  """
46
63
  Raised when a request to the DataMasque server fails before any response is received.
@@ -30,7 +30,7 @@ class FileClient(BaseClient):
30
30
  response = self.make_request(
31
31
  "POST",
32
32
  file_type.get_url(),
33
- data={"name": file_name},
33
+ data={"name": file_name, **file_type.get_extra_form_data()},
34
34
  files=[
35
35
  UploadFile(
36
36
  field_name=file_type.get_content_param_name(),
@@ -1,12 +1,15 @@
1
1
  """Typed request and response shapes for schema-discovery and ruleset-generation endpoints."""
2
2
 
3
+ from enum import Enum
3
4
  from typing import Any, Optional, Union
4
5
 
5
- from pydantic import BaseModel, ConfigDict, Field, field_validator
6
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
6
7
 
7
8
  from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id
8
9
  from datamasque.client.models.data_selection import HashColumnsTableConfig, Locator, UserSelection
10
+ from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, unwrap_discovery_config_id
9
11
  from datamasque.client.models.pagination import Page
12
+ from datamasque.client.models.runs import RunConnectionRef
10
13
 
11
14
 
12
15
  class InDataDiscoveryRule(BaseModel):
@@ -27,15 +30,17 @@ class InDataDiscoveryConfig(BaseModel):
27
30
  row_sample_size: Optional[int] = None
28
31
  custom_rules: Optional[list[InDataDiscoveryRule]] = None
29
32
  non_sensitive_rules: Optional[list[InDataDiscoveryRule]] = None
33
+ ignore_rules: Optional[list[InDataDiscoveryRule]] = None
30
34
  force: Optional[bool] = None
31
35
 
32
36
 
33
37
  class SchemaDiscoveryRequest(BaseModel):
34
38
  """
35
- Request body for `POST /api/schema-discovery/`.
39
+ Request body for `POST /api/schema-discovery/` (the keyword-driven schema-discovery trigger).
36
40
 
37
- `connection` accepts either a `ConnectionId` or a full `ConnectionConfig` returned by an earlier client call.
38
- Every other field uses the server's default value when omitted.
41
+ `connection` accepts either a `ConnectionId` or a full `ConnectionConfig`
42
+ returned by an earlier client call.
43
+ This request does not accept a `discovery_config`.
39
44
  """
40
45
 
41
46
  model_config = ConfigDict(extra="forbid")
@@ -54,6 +59,45 @@ class SchemaDiscoveryRequest(BaseModel):
54
59
  def _unwrap_connection(cls, value: Any) -> Any:
55
60
  return unwrap_connection_id(value)
56
61
 
62
+ @model_validator(mode="before")
63
+ @classmethod
64
+ def _reject_discovery_config(cls, data: Any) -> Any:
65
+ if isinstance(data, dict) and "discovery_config" in data:
66
+ raise ValueError(
67
+ "`discovery_config` is not accepted by the keyword-driven schema-discovery request; "
68
+ "use `start_schema_discovery_run_from_config` with a `SchemaDiscoveryFromConfigRequest` "
69
+ "to run from a saved discovery config."
70
+ )
71
+ return data
72
+
73
+
74
+ class SchemaDiscoveryFromConfigRequest(BaseModel):
75
+ """
76
+ Request body for `POST /api/schema-discovery/v2/` (start a run from a saved discovery config).
77
+
78
+ `connection` accepts either a `ConnectionId` or a full `ConnectionConfig`
79
+ returned by an earlier client call.
80
+ `discovery_config` is required: pass a `DiscoveryConfigId`, a full `DiscoveryConfig`,
81
+ or `None` to run with the default discovery options.
82
+ `schemas` optionally scopes the run to specific schemas; omit it to scan the connection's default schema.
83
+ """
84
+
85
+ model_config = ConfigDict(extra="forbid")
86
+
87
+ connection: Union[ConnectionId, ConnectionConfig]
88
+ discovery_config: Optional[Union[DiscoveryConfigId, DiscoveryConfig]]
89
+ schemas: Optional[list[str]] = None
90
+
91
+ @field_validator("connection", mode="before")
92
+ @classmethod
93
+ def _unwrap_connection(cls, value: Any) -> Any:
94
+ return unwrap_connection_id(value)
95
+
96
+ @field_validator("discovery_config", mode="before")
97
+ @classmethod
98
+ def _unwrap_discovery_config(cls, value: Any) -> Any:
99
+ return unwrap_discovery_config_id(value)
100
+
57
101
 
58
102
  class RulesetGenerationRequest(BaseModel):
59
103
  """
@@ -77,6 +121,114 @@ class RulesetGenerationRequest(BaseModel):
77
121
  return unwrap_connection_id(value)
78
122
 
79
123
 
124
+ class FileFilterMatchAgainst(Enum):
125
+ """Which part of a file's path an `include`/`skip` filter is matched against."""
126
+
127
+ path = "path"
128
+ filename = "filename"
129
+
130
+
131
+ class FileFilter(BaseModel):
132
+ """
133
+ A single `include` or `skip` filter for file data discovery.
134
+
135
+ Exactly one of `glob` or `regex` must be set.
136
+ `match_against` selects whether the pattern is applied to the full path or just the filename
137
+ (defaults to the full path when omitted).
138
+ """
139
+
140
+ model_config = ConfigDict(extra="forbid")
141
+
142
+ glob: Optional[str] = Field(default=None, min_length=1)
143
+ regex: Optional[str] = Field(default=None, min_length=1)
144
+ match_against: Optional[FileFilterMatchAgainst] = None
145
+
146
+ @model_validator(mode="after")
147
+ def _check_glob_xor_regex(self) -> "FileFilter":
148
+ if (self.glob is None) == (self.regex is None):
149
+ raise ValueError("A `FileFilter` must set exactly one of `glob` or `regex`.")
150
+ return self
151
+
152
+
153
+ class FileDataDiscoveryOptions(BaseModel):
154
+ """Run options nested under `FileDataDiscoveryRequest.options`."""
155
+
156
+ model_config = ConfigDict(extra="forbid")
157
+
158
+ diagnostic_logging: Optional[bool] = None
159
+
160
+
161
+ class FileDataDiscoveryRequest(BaseModel):
162
+ """
163
+ Request body for `POST /api/run-file-data-discovery/` (the keyword-driven file-data-discovery trigger).
164
+
165
+ `connection` accepts either a `ConnectionId` or a full `ConnectionConfig`
166
+ returned by an earlier client call.
167
+ This request does not accept a `discovery_config`.
168
+ """
169
+
170
+ model_config = ConfigDict(extra="forbid")
171
+
172
+ connection: Union[ConnectionId, ConnectionConfig]
173
+ options: Optional[FileDataDiscoveryOptions] = None
174
+ custom_keywords: list[str] = Field(default_factory=list)
175
+ ignored_keywords: list[str] = Field(default_factory=list)
176
+ disable_built_in_keywords: bool = False
177
+ disable_global_custom_keywords: Optional[bool] = None
178
+ disable_global_ignored_keywords: Optional[bool] = None
179
+ in_data_discovery: Optional[InDataDiscoveryConfig] = None
180
+ recurse: Optional[bool] = None
181
+ include: Optional[list[FileFilter]] = None
182
+ skip: Optional[list[FileFilter]] = None
183
+ encoding: Optional[str] = None
184
+ workers: Optional[int] = None
185
+
186
+ @field_validator("connection", mode="before")
187
+ @classmethod
188
+ def _unwrap_connection(cls, value: Any) -> Any:
189
+ return unwrap_connection_id(value)
190
+
191
+ @model_validator(mode="before")
192
+ @classmethod
193
+ def _reject_discovery_config(cls, data: Any) -> Any:
194
+ if isinstance(data, dict) and "discovery_config" in data:
195
+ raise ValueError(
196
+ "`discovery_config` is not accepted by the keyword-driven file-data-discovery request; "
197
+ "use `start_file_data_discovery_run_from_config` with a `FileDataDiscoveryFromConfigRequest` "
198
+ "to run from a saved discovery config."
199
+ )
200
+ return data
201
+
202
+
203
+ class FileDataDiscoveryFromConfigRequest(BaseModel):
204
+ """
205
+ Request body for `POST /api/run-file-data-discovery/v2/` (start a run from a saved discovery config).
206
+
207
+ `connection` accepts either a `ConnectionId` or a full `ConnectionConfig`
208
+ returned by an earlier client call.
209
+ `discovery_config` is required: pass a `DiscoveryConfigId`, a full `DiscoveryConfig`,
210
+ or `None` to run with the server's default discovery options.
211
+ `options` carries the `diagnostic_logging` run-time toggle;
212
+ detection and file-handling settings come from the discovery config, not the request.
213
+ """
214
+
215
+ model_config = ConfigDict(extra="forbid")
216
+
217
+ connection: Union[ConnectionId, ConnectionConfig]
218
+ discovery_config: Optional[Union[DiscoveryConfigId, DiscoveryConfig]]
219
+ options: Optional[FileDataDiscoveryOptions] = None
220
+
221
+ @field_validator("connection", mode="before")
222
+ @classmethod
223
+ def _unwrap_connection(cls, value: Any) -> Any:
224
+ return unwrap_connection_id(value)
225
+
226
+ @field_validator("discovery_config", mode="before")
227
+ @classmethod
228
+ def _unwrap_discovery_config(cls, value: Any) -> Any:
229
+ return unwrap_discovery_config_id(value)
230
+
231
+
80
232
  class FileRulesetGenerationRequest(BaseModel):
81
233
  """
82
234
  Request body for `POST /api/generate-file-ruleset/`.
@@ -189,11 +341,11 @@ class FileDiscoveryMatch(BaseModel):
189
341
 
190
342
  model_config = ConfigDict(extra="allow")
191
343
 
192
- categories: Optional[list[str]] = None
193
- flagged_by: Optional[str] = None
194
- description: Optional[str] = None
195
- label: Optional[str] = None
196
- hit_ratio: Optional[int] = None
344
+ flagged_by: str
345
+ description: str
346
+ label: Optional[str] = None # Omitted for non-sensitive and ignored matches.
347
+ categories: Optional[list[str]] = None # Omitted for ignored matches.
348
+ hit_ratio: Optional[int] = None # None for metadata matches, percentage 0-100 for IDD matches.
197
349
 
198
350
 
199
351
  class FileDiscoveryLocatorResult(BaseModel):
@@ -201,9 +353,9 @@ class FileDiscoveryLocatorResult(BaseModel):
201
353
 
202
354
  model_config = ConfigDict(extra="allow")
203
355
 
204
- locator: Optional[Locator] = None
205
- matches: Optional[list[FileDiscoveryMatch]] = None
206
- data_types: Optional[list[str]] = None
356
+ locator: Locator
357
+ matches: list[FileDiscoveryMatch]
358
+ data_types: list[str]
207
359
 
208
360
 
209
361
  class FileDiscoveryFile(BaseModel):
@@ -211,8 +363,8 @@ class FileDiscoveryFile(BaseModel):
211
363
 
212
364
  model_config = ConfigDict(extra="allow")
213
365
 
214
- path: Optional[str] = None
215
- file_type: Optional[str] = None
366
+ path: str
367
+ file_type: str
216
368
  delimiter: Optional[str] = None
217
369
  encoding: Optional[str] = None
218
370
 
@@ -222,8 +374,8 @@ class FileDiscoveryResult(BaseModel):
222
374
 
223
375
  model_config = ConfigDict(extra="allow")
224
376
 
225
- id: Optional[int] = None
226
- connection: Optional[Any] = None
227
- file_type: Optional[str] = None
228
- files: Optional[list[FileDiscoveryFile]] = None
229
- results: Optional[list[FileDiscoveryLocatorResult]] = None
377
+ id: int
378
+ connection: RunConnectionRef
379
+ file_type: str
380
+ files: list[FileDiscoveryFile]
381
+ results: list[FileDiscoveryLocatorResult]
@@ -0,0 +1,53 @@
1
+ import enum
2
+ from datetime import datetime
3
+ from typing import Any, NewType, Optional
4
+
5
+ from pydantic import BaseModel, ConfigDict, Field
6
+
7
+ from datamasque.client.models.status import ValidationStatus
8
+
9
+ DiscoveryConfigId = NewType("DiscoveryConfigId", str)
10
+
11
+
12
+ class DiscoveryConfigType(enum.Enum):
13
+ """Which discovery config variant a config targets: database (qualified columns) or file (locators)."""
14
+
15
+ database = "database"
16
+ file = "file"
17
+
18
+
19
+ def unwrap_discovery_config_id(value: Any) -> Any:
20
+ """
21
+ Coerce a `DiscoveryConfig` to its `id`; pass other values through unchanged.
22
+
23
+ Used by request-model validators that accept either a `DiscoveryConfigId`
24
+ or a full `DiscoveryConfig` for user convenience.
25
+ Raises `ValueError` if the config has no `id`
26
+ (i.e. the caller hasn't yet created it on the server).
27
+ """
28
+
29
+ if isinstance(value, DiscoveryConfig):
30
+ if value.id is None:
31
+ raise ValueError("Discovery config has not been created yet (id is None)")
32
+ return value.id
33
+
34
+ return value
35
+
36
+
37
+ class DiscoveryConfig(BaseModel):
38
+ """Represents a named, persisted YAML discovery configuration."""
39
+
40
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
41
+
42
+ name: str
43
+ yaml: Optional[str] = Field(default=None, alias="config_yaml")
44
+ config_type: DiscoveryConfigType
45
+
46
+ # Server-populated read-only fields, excluded from request bodies.
47
+ id: Optional[DiscoveryConfigId] = Field(default=None, exclude=True)
48
+ is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True)
49
+ """Validation status; may be `in_progress` briefly after creating a large config."""
50
+ validation_error: Optional[str] = Field(default=None, exclude=True)
51
+ """Human-readable validation error, or `None` when valid."""
52
+ created: Optional[datetime] = Field(default=None, exclude=True)
53
+ modified: Optional[datetime] = Field(default=None, exclude=True)
@@ -40,6 +40,12 @@ class DataMasqueFile(BaseModel):
40
40
 
41
41
  raise NotImplementedError # pragma: no cover
42
42
 
43
+ @classmethod
44
+ def get_extra_form_data(cls) -> dict[str, str]:
45
+ """Extra multipart form fields to send alongside the file on upload. Empty by default."""
46
+
47
+ return {}
48
+
43
49
 
44
50
  class SeedFile(DataMasqueFile):
45
51
  """Represents a seed file (CSV file)."""
@@ -76,6 +82,11 @@ class SslZipFile(DataMasqueFile):
76
82
  def get_content_param_name(cls) -> str:
77
83
  return "zip_archive"
78
84
 
85
+ @classmethod
86
+ def get_extra_form_data(cls) -> dict[str, str]:
87
+ # The connection-filesets endpoint requires a database_type; SSL filesets are MySQL-only today.
88
+ return {"database_type": "mysql"}
89
+
79
90
 
80
91
  class SnowflakeKeyFile(DataMasqueFile):
81
92
  """Represents a private SSH key file for Snowflake connections."""
@@ -0,0 +1,60 @@
1
+ from collections.abc import Mapping
2
+ from datetime import datetime
3
+ from typing import Any, Optional
4
+
5
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
6
+
7
+
8
+ class GitSnapshot(BaseModel):
9
+ """
10
+ Git provenance for a ruleset or ruleset library.
11
+
12
+ Identifies the commit the entity's contents came from —
13
+ `commit_sha` on `branch` in `repo_url`, as of `synced_at`.
14
+ """
15
+
16
+ branch: str
17
+ commit_sha: str
18
+ repo_url: str
19
+ synced_at: datetime
20
+
21
+
22
+ GIT_RESPONSE_FIELDS = ("git_branch", "git_commit_sha", "git_repo_url", "git_synced_at")
23
+
24
+
25
+ def git_snapshot_from_response(data: Mapping[str, Any]) -> Optional[GitSnapshot]:
26
+ """Build a `GitSnapshot` from the server's flat `git_*` fields, or `None` when not git-synced."""
27
+ if data.get("git_branch") is None:
28
+ return None
29
+
30
+ return GitSnapshot(
31
+ branch=data["git_branch"],
32
+ commit_sha=data["git_commit_sha"],
33
+ repo_url=data["git_repo_url"],
34
+ synced_at=data["git_synced_at"],
35
+ )
36
+
37
+
38
+ class GitTrackedEntity(BaseModel):
39
+ """Base class for rulesets and ruleset libraries that carry git provenance."""
40
+
41
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
42
+
43
+ git: Optional[GitSnapshot] = Field(default=None, exclude=True)
44
+ """Git provenance, or `None` when the entity is not currently in sync with a git commit."""
45
+
46
+ @model_validator(mode="before")
47
+ @classmethod
48
+ def _collapse_git_fields(cls, data: Any) -> Any:
49
+ if not isinstance(data, dict) or "git" in data:
50
+ return data
51
+
52
+ data = dict(data)
53
+ snapshot = git_snapshot_from_response(data)
54
+ for field in GIT_RESPONSE_FIELDS:
55
+ data.pop(field, None)
56
+
57
+ if snapshot is not None:
58
+ data["git"] = snapshot
59
+
60
+ return data
@@ -1,9 +1,10 @@
1
1
  import enum
2
2
  from typing import Any, NewType, Optional
3
3
 
4
- from pydantic import BaseModel, ConfigDict, Field
4
+ from pydantic import Field
5
5
 
6
- from datamasque.client.models.status import ValidationStatus
6
+ from datamasque.client.models.git import GitTrackedEntity
7
+ from datamasque.client.models.status import ValidationErrorType, ValidationStatus
7
8
 
8
9
  RulesetId = NewType("RulesetId", str)
9
10
 
@@ -33,13 +34,17 @@ class RulesetType(enum.Enum):
33
34
  database = "database"
34
35
 
35
36
 
36
- class Ruleset(BaseModel):
37
+ class Ruleset(GitTrackedEntity):
37
38
  """Represents a ruleset."""
38
39
 
39
- model_config = ConfigDict(extra="allow", populate_by_name=True)
40
-
41
40
  name: str
42
41
  yaml: str = Field(default="", alias="config_yaml")
43
42
  ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type")
44
- id: Optional[RulesetId] = None
45
- is_valid: Optional[ValidationStatus] = None
43
+
44
+ # Server-populated read-only fields, excluded from request bodies.
45
+ id: Optional[RulesetId] = Field(default=None, exclude=True)
46
+ is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True)
47
+ validation_error: Optional[str] = Field(default=None, exclude=True)
48
+ """Human-readable validation error, or `None` when valid."""
49
+ validation_error_type: Optional[ValidationErrorType] = Field(default=None, exclude=True)
50
+ """Category of the validation failure, or `None` when valid."""
@@ -1,22 +1,25 @@
1
1
  from datetime import datetime
2
2
  from typing import NewType, Optional
3
3
 
4
- from pydantic import BaseModel, ConfigDict, Field
4
+ from pydantic import Field
5
5
 
6
+ from datamasque.client.models.git import GitTrackedEntity
6
7
  from datamasque.client.models.status import ValidationStatus
7
8
 
8
9
  RulesetLibraryId = NewType("RulesetLibraryId", str)
9
10
 
10
11
 
11
- class RulesetLibrary(BaseModel):
12
+ class RulesetLibrary(GitTrackedEntity):
12
13
  """Represents a ruleset library."""
13
14
 
14
- model_config = ConfigDict(extra="allow", populate_by_name=True)
15
-
16
15
  name: str
17
16
  namespace: str = ""
18
17
  yaml: Optional[str] = Field(default=None, alias="config_yaml")
19
- id: Optional[RulesetLibraryId] = None
20
- is_valid: Optional[ValidationStatus] = None
21
- created: Optional[datetime] = None
22
- modified: Optional[datetime] = None
18
+
19
+ # Server-populated read-only fields, excluded from request bodies.
20
+ id: Optional[RulesetLibraryId] = Field(default=None, exclude=True)
21
+ is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True)
22
+ validation_error: Optional[str] = Field(default=None, exclude=True)
23
+ """Human-readable validation error, or `None` when valid."""
24
+ created: Optional[datetime] = Field(default=None, exclude=True)
25
+ modified: Optional[datetime] = Field(default=None, exclude=True)
@@ -30,6 +30,7 @@ class MaskingRunOptions(BaseModel):
30
30
  if supplied,
31
31
  must be 16–256 characters and is used as the per-run encryption key;
32
32
  the server auto-generates one when omitted.
33
+ Set `auto_pull` to refresh the run's ruleset from git before the run starts.
33
34
  """
34
35
 
35
36
  model_config = ConfigDict(extra="forbid")
@@ -41,6 +42,10 @@ class MaskingRunOptions(BaseModel):
41
42
  diagnostic_logging: Optional[bool] = None
42
43
  run_secret: Optional[str] = Field(default=None, min_length=16, max_length=256)
43
44
  disable_instance_secret: Optional[bool] = None
45
+ auto_pull: Optional[bool] = None
46
+ """When `True`, pull the run's ruleset from git before starting."""
47
+ auto_pull_branch: Optional[str] = None
48
+ """Branch to auto-pull from; omitted or empty uses the instance's configured branch."""
44
49
 
45
50
 
46
51
  class MaskingRunRequest(BaseModel):
@@ -50,6 +55,9 @@ class MaskingRunRequest(BaseModel):
50
55
  `connection`, `destination_connection`, and `ruleset` accept either the server-assigned ID
51
56
  or the corresponding object returned by an earlier client call (e.g. a `ConnectionConfig`
52
57
  or `Ruleset`); the object's `id` is extracted at construction time.
58
+
59
+ Set `is_user_subscribed=True` to subscribe the requesting user to the run's email notifications;
60
+ when omitted the server applies its default (no subscription).
53
61
  """
54
62
 
55
63
  model_config = ConfigDict(extra="forbid")
@@ -60,6 +68,7 @@ class MaskingRunRequest(BaseModel):
60
68
  destination_connection: Optional[Union[ConnectionId, ConnectionConfig]] = None
61
69
  options: MaskingRunOptions = Field(default_factory=MaskingRunOptions)
62
70
  name: Optional[str] = None
71
+ is_user_subscribed: Optional[bool] = None
63
72
 
64
73
  @field_validator("connection", "destination_connection", mode="before")
65
74
  @classmethod
@@ -10,6 +10,15 @@ class ValidationStatus(enum.Enum):
10
10
  unknown = "unknown"
11
11
 
12
12
 
13
+ class ValidationErrorType(enum.Enum):
14
+ """Categorises why a ruleset failed validation (see `Ruleset.validation_error_type`)."""
15
+
16
+ ruleset = "ruleset"
17
+ library_missing = "library_missing"
18
+ library_invalid = "library_invalid"
19
+ expansion = "expansion" # The ruleset is not valid once its library references are expanded.
20
+
21
+
13
22
  class MaskingRunStatus(enum.Enum):
14
23
  """List of valid masking run statuses."""
15
24
 
@@ -63,7 +63,8 @@ class RulesetLibraryClient(BaseClient):
63
63
  """
64
64
  Creates a new ruleset library on the server.
65
65
 
66
- Sets the library's server-assigned fields (`id`, `is_valid`, `created`, `modified`) and returns the library.
66
+ Sets the library's server-assigned fields
67
+ (`id`, `is_valid`, `validation_error`, `git`, `created`, `modified`) and returns the library.
67
68
  """
68
69
 
69
70
  data = library.model_dump(exclude_none=True, by_alias=True, mode="json")
@@ -71,6 +72,8 @@ class RulesetLibraryClient(BaseClient):
71
72
  created_library = RulesetLibrary.model_validate(response.json())
72
73
  library.id = created_library.id
73
74
  library.is_valid = created_library.is_valid
75
+ library.validation_error = created_library.validation_error
76
+ library.git = created_library.git
74
77
  library.created = created_library.created
75
78
  library.modified = created_library.modified
76
79
  logger.info('Creation of ruleset library "%s" successful', library.name)
@@ -90,6 +93,8 @@ class RulesetLibraryClient(BaseClient):
90
93
  response = self.make_request("PUT", f"/api/ruleset-libraries/{library.id}/", data=data)
91
94
  updated_library = RulesetLibrary.model_validate(response.json())
92
95
  library.is_valid = updated_library.is_valid
96
+ library.validation_error = updated_library.validation_error
97
+ library.git = updated_library.git
93
98
  library.modified = updated_library.modified
94
99
  logger.debug('Update of ruleset library "%s" successful', library.name)
95
100
  return library
@@ -110,7 +115,7 @@ class RulesetLibraryClient(BaseClient):
110
115
 
111
116
  def delete_ruleset_library_by_id_if_exists(self, library_id: RulesetLibraryId, *, force: bool = False) -> None:
112
117
  """
113
- Deletes (archives) the ruleset library with the given ID.
118
+ Deletes the ruleset library with the given ID.
114
119
 
115
120
  No-op if the library does not exist.
116
121
 
@@ -139,13 +144,13 @@ class RulesetLibraryClient(BaseClient):
139
144
  self.delete_ruleset_library_by_id_if_exists(lib.id, force=force)
140
145
 
141
146
  def iter_rulesets_using_library(self, library_id: RulesetLibraryId) -> Iterator[Ruleset]:
142
- """Lazily iterate non-archived rulesets that import the given library."""
147
+ """Lazily iterate rulesets that import the given library."""
143
148
 
144
149
  return self._iter_paginated(f"/api/ruleset-libraries/{library_id}/rulesets/", model=Ruleset)
145
150
 
146
151
  def list_rulesets_using_library(self, library_id: RulesetLibraryId) -> list[Ruleset]:
147
152
  """
148
- Lists non-archived rulesets that import the given library.
153
+ Lists rulesets that import the given library.
149
154
 
150
155
  Note: The YAML content is not included in the response for performance.
151
156
  Each returned Ruleset will have an empty string for `yaml`.
@@ -2,8 +2,7 @@ import logging
2
2
 
3
3
  from datamasque.client.base import BaseClient
4
4
  from datamasque.client.exceptions import DataMasqueException
5
- from datamasque.client.models.ruleset import Ruleset, RulesetId
6
- from datamasque.client.models.status import ValidationStatus
5
+ from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType
7
6
 
8
7
  logger = logging.getLogger(__name__)
9
8
 
@@ -21,17 +20,18 @@ class RulesetClient(BaseClient):
21
20
  """
22
21
  Creates or updates a ruleset.
23
22
 
24
- Populates the given ruleset's `id` and `is_valid` fields from the server response,
25
- and returns the same ruleset instance for convenience.
23
+ Populates the given ruleset's `id`, `is_valid`, `validation_error`, `validation_error_type`,
24
+ and `git` fields from the server response, and returns the same ruleset instance for convenience.
26
25
  """
27
26
 
28
27
  data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json")
29
28
  response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"})
30
- response_data = response.json()
31
- ruleset.id = RulesetId(response_data["id"])
32
- is_valid = response_data.get("is_valid")
33
- if is_valid is not None:
34
- ruleset.is_valid = ValidationStatus(is_valid)
29
+ created = Ruleset.model_validate(response.json())
30
+ ruleset.id = created.id
31
+ ruleset.is_valid = created.is_valid
32
+ ruleset.validation_error = created.validation_error
33
+ ruleset.validation_error_type = created.validation_error_type
34
+ ruleset.git = created.git
35
35
 
36
36
  if response.status_code == 201:
37
37
  logger.info('Creation of ruleset "%s" successful', ruleset.name)
@@ -45,12 +45,20 @@ class RulesetClient(BaseClient):
45
45
 
46
46
  self._delete_if_exists(f"/api/rulesets/{ruleset_id}/")
47
47
 
48
- def delete_ruleset_by_name_if_exists(self, ruleset_name: str) -> None:
49
- """Deletes the ruleset with the given name. No-op if the ruleset does not exist."""
48
+ def delete_ruleset_by_name_if_exists(self, ruleset_name: str, ruleset_type: RulesetType) -> None:
49
+ """
50
+ Deletes the ruleset with the given name and type.
51
+
52
+ Ruleset names are unique per type, so a type is required to identify a single ruleset.
53
+ No-op if no such ruleset exists.
54
+ """
50
55
 
51
- all_rulesets = self.list_rulesets()
52
- rulesets_matching_name = [ruleset for ruleset in all_rulesets if ruleset.name == ruleset_name]
53
- for ruleset in rulesets_matching_name:
56
+ matching = [
57
+ ruleset
58
+ for ruleset in self.list_rulesets()
59
+ if ruleset.name == ruleset_name and ruleset.ruleset_type is ruleset_type
60
+ ]
61
+ for ruleset in matching:
54
62
  if ruleset.id is None:
55
63
  raise DataMasqueException(f'Server returned a ruleset named "{ruleset.name}" without an `id`.')
56
64
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datamasque-python
3
- Version: 1.0.5
3
+ Version: 1.1.0
4
4
  Summary: Official Python client for the DataMasque data-masking API.
5
5
  Project-URL: Homepage, https://datamasque.com/
6
6
  Project-URL: Documentation, https://datamasque-python.readthedocs.io/
@@ -0,0 +1,36 @@
1
+ datamasque/client/__init__.py,sha256=GmpVFNWZPu6jyadhY6SOZp4hLR1wUc4cgzwLcqoEA5o,6312
2
+ datamasque/client/base.py,sha256=vP1LuYbqq6U8NAEJWnwNrdt_Fa2wjQTJXHf0gSFO79c,14115
3
+ datamasque/client/connections.py,sha256=EFinx8fJRme0mTxuWY3d29UnmUFbsQhMaUQT0Ma2PK4,2885
4
+ datamasque/client/discovery.py,sha256=NluzWjC-QGzxWEHr6zMjKDkhx-Z9w5M0IBcR258OEhY,20536
5
+ datamasque/client/discovery_configs.py,sha256=G-W-X_KizjjQLa-Rb6ixLjlkxbkCe4WGCOfYZrGtxIk,6420
6
+ datamasque/client/dmclient.py,sha256=tA7V8GEQmnsalTHUZch8YmIUynYw2ldK-mmUGbofFL4,1650
7
+ datamasque/client/exceptions.py,sha256=Tx7tE6i1X_oemR3EOJC97_sihZiToHqGuOQmWWxs_BE,3131
8
+ datamasque/client/files.py,sha256=vc2tZgT8RdwuCObIQ7H-Qjb80H1PBFqD0nG5Y1tOA_o,3503
9
+ datamasque/client/ifm.py,sha256=uIMxpLIPvDiDO1m4bxezixNIUFIFY0MXWRMQNvTbpTA,11858
10
+ datamasque/client/license.py,sha256=pluYaSU168OC6_laB9bM9H3Vuuxs8wa9gujCJvtoJCk,1392
11
+ datamasque/client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ datamasque/client/ruleset_libraries.py,sha256=bMUMBHdxjDyFOziw9LMRy5xChPvHvCjpjAUoVeTm1e0,7086
13
+ datamasque/client/rulesets.py,sha256=Fsnsv-uAhF1DBnLI-pRpOrWJ7aCSZspcuU1r0RYTzUc,2655
14
+ datamasque/client/runs.py,sha256=ZPSkkuyqMiwy7dLbWZ1PAEKaesKLeNRX7xEJGfzieVg,7427
15
+ datamasque/client/settings.py,sha256=Ui8AyR2XdoW8MZ9FIrGn2jm8DLzUGWIL8i2DOTvu7hc,2898
16
+ datamasque/client/users.py,sha256=VCUo2CJyOw4-aO_3mp_w_0BcSoa6-h1HcReMcAMyumw,3701
17
+ datamasque/client/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
18
+ datamasque/client/models/connection.py,sha256=TSv-bbOFlYCRBZHJiRgiwNuN0es_GdJd-_L3LUsQ-dI,15773
19
+ datamasque/client/models/data_selection.py,sha256=406yyUZ5NmLBSql2lYM1gsTWY5GWmnutmF-DjznAoLc,1904
20
+ datamasque/client/models/discovery.py,sha256=wEveTc8vh6dOJx6K2rkbdbU_e1EbI8jSRhZ5SgDF2wY,13225
21
+ datamasque/client/models/discovery_config.py,sha256=1CHMgSxMXuarPGfRcz9wX7jeZLn1SDw0UEe-L5naE54,1917
22
+ datamasque/client/models/dm_instance.py,sha256=yjjpHZJTFhJp3lAinTEffgKrxnadrJys1EuhO77wQcA,1561
23
+ datamasque/client/models/files.py,sha256=7hG3Q1-XYb1PF88l3ppsmBp2tjNtvQvv-RNTrOa7Nts,2784
24
+ datamasque/client/models/git.py,sha256=F0KdMFe2vMyL-9tp3SGKGeOlbD1gbcjQn0jpOzNzybc,1782
25
+ datamasque/client/models/ifm.py,sha256=j0Ef2BZYTk6MNZO6HS6mW0qWs1_wWD9PlUP_Y6WyBQI,5563
26
+ datamasque/client/models/license.py,sha256=OqIn4Sx3ATBStgt5KNiCOBChTTFYLBsyGj7inAl8oB4,2113
27
+ datamasque/client/models/pagination.py,sha256=egg9aO2cf6KUDwDANPu5RxpJbdRKdwUdCQAtusnuf1c,651
28
+ datamasque/client/models/ruleset.py,sha256=ZYv6SmcrONV5rLK0tKjhlKCwZiCU1dMYtH7olxI4l5A,1678
29
+ datamasque/client/models/ruleset_library.py,sha256=5UbL_M47mgPedo8xt4q5crUh4aeOVoEMAEKbGO8CfiY,950
30
+ datamasque/client/models/runs.py,sha256=UtPMGCJFLFP1f2nyVGAbl3A5_wAONrRNJ63y43Hbpi0,6213
31
+ datamasque/client/models/status.py,sha256=LuTpgocedVqeU0zxuoyJ7LsxMgHtSkrwHVMYaX_7QGg,2320
32
+ datamasque/client/models/user.py,sha256=UGAUzgJkf78m24_zFXXoA99zdut48BXkX_ivV8yq1Vc,2043
33
+ datamasque_python-1.1.0.dist-info/METADATA,sha256=xZqEPFC-2faxpi_P_8wKBIRWNqKpZuEQdFWjvhWj26Q,4187
34
+ datamasque_python-1.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
35
+ datamasque_python-1.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
36
+ datamasque_python-1.1.0.dist-info/RECORD,,
@@ -1,33 +0,0 @@
1
- datamasque/client/__init__.py,sha256=rH7hQyA_nFVkv8eV_C1Ds_40v9xzCK__uXIJtWMLfuQ,5472
2
- datamasque/client/base.py,sha256=vP1LuYbqq6U8NAEJWnwNrdt_Fa2wjQTJXHf0gSFO79c,14115
3
- datamasque/client/connections.py,sha256=EFinx8fJRme0mTxuWY3d29UnmUFbsQhMaUQT0Ma2PK4,2885
4
- datamasque/client/discovery.py,sha256=uA8h6vRqsxSAzSAV9bebJwU47XINlaVy7V1nTYDiaCM,12634
5
- datamasque/client/dmclient.py,sha256=OPYMzc57gUHPs6iL_J2DYp06MfOaabpTlISmnNCpqS4,1553
6
- datamasque/client/exceptions.py,sha256=F9FYCxP-ERXkVD1L3yh_rWqcW3IsPL-c-Ic4qMYoGnw,2542
7
- datamasque/client/files.py,sha256=5Gzel4aLby8T7ncOIV6wtgVbFEk0eaEy7wxohh9u0zE,3468
8
- datamasque/client/ifm.py,sha256=uIMxpLIPvDiDO1m4bxezixNIUFIFY0MXWRMQNvTbpTA,11858
9
- datamasque/client/license.py,sha256=pluYaSU168OC6_laB9bM9H3Vuuxs8wa9gujCJvtoJCk,1392
10
- datamasque/client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
- datamasque/client/ruleset_libraries.py,sha256=tyN--cndzG0gwFnHY9fDtObTLzGsK0wrV5lxxjRP72g,6868
12
- datamasque/client/rulesets.py,sha256=2Zh6QUihZ8p3dNT6QbaWuxvMk-Whp67JUj8UxqeRYUY,2407
13
- datamasque/client/runs.py,sha256=ZPSkkuyqMiwy7dLbWZ1PAEKaesKLeNRX7xEJGfzieVg,7427
14
- datamasque/client/settings.py,sha256=Ui8AyR2XdoW8MZ9FIrGn2jm8DLzUGWIL8i2DOTvu7hc,2898
15
- datamasque/client/users.py,sha256=VCUo2CJyOw4-aO_3mp_w_0BcSoa6-h1HcReMcAMyumw,3701
16
- datamasque/client/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
- datamasque/client/models/connection.py,sha256=TSv-bbOFlYCRBZHJiRgiwNuN0es_GdJd-_L3LUsQ-dI,15773
18
- datamasque/client/models/data_selection.py,sha256=406yyUZ5NmLBSql2lYM1gsTWY5GWmnutmF-DjznAoLc,1904
19
- datamasque/client/models/discovery.py,sha256=BawKusPuhyt0gRRnWKYf-ZKF7V54GxkhgYIrEQ6XkOs,7283
20
- datamasque/client/models/dm_instance.py,sha256=yjjpHZJTFhJp3lAinTEffgKrxnadrJys1EuhO77wQcA,1561
21
- datamasque/client/models/files.py,sha256=OaWVD-AX77vvaN_qxB6Uzn2HRasb7wNo-6QAvRzV3og,2381
22
- datamasque/client/models/ifm.py,sha256=j0Ef2BZYTk6MNZO6HS6mW0qWs1_wWD9PlUP_Y6WyBQI,5563
23
- datamasque/client/models/license.py,sha256=OqIn4Sx3ATBStgt5KNiCOBChTTFYLBsyGj7inAl8oB4,2113
24
- datamasque/client/models/pagination.py,sha256=egg9aO2cf6KUDwDANPu5RxpJbdRKdwUdCQAtusnuf1c,651
25
- datamasque/client/models/ruleset.py,sha256=tf5j9ih3B0uEF4yPTnyCsLblFWzH-t9KnvJWuyJaiCs,1256
26
- datamasque/client/models/ruleset_library.py,sha256=gIqb6yn4-f9i2OydOLk1cd0zId9nGv5l_yUZwFHZ1aA,652
27
- datamasque/client/models/runs.py,sha256=oVYo9jp9s5LjWts0LKsYpX3HUBmrVwuuzDqFtQmTjvo,5673
28
- datamasque/client/models/status.py,sha256=rjH6YSwAHoOUaUCADwvMhuKd0ygT0c2w2Ek5SV8PWD8,1984
29
- datamasque/client/models/user.py,sha256=UGAUzgJkf78m24_zFXXoA99zdut48BXkX_ivV8yq1Vc,2043
30
- datamasque_python-1.0.5.dist-info/METADATA,sha256=U0jpV668Y8w1fBtPcx4Rh5ZwzT1cbptYAepZkJFhBAA,4187
31
- datamasque_python-1.0.5.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
32
- datamasque_python-1.0.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
33
- datamasque_python-1.0.5.dist-info/RECORD,,