great-expectations-cloud 20250903.0.dev0__py3-none-any.whl → 20250904.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- great_expectations_cloud/agent/actions/utils.py +13 -4
- great_expectations_cloud/agent/agent.py +24 -50
- great_expectations_cloud/agent/models.py +0 -1
- {great_expectations_cloud-20250903.0.dev0.dist-info → great_expectations_cloud-20250904.0.dev1.dist-info}/METADATA +1 -1
- {great_expectations_cloud-20250903.0.dev0.dist-info → great_expectations_cloud-20250904.0.dev1.dist-info}/RECORD +8 -8
- {great_expectations_cloud-20250903.0.dev0.dist-info → great_expectations_cloud-20250904.0.dev1.dist-info}/LICENSE +0 -0
- {great_expectations_cloud-20250903.0.dev0.dist-info → great_expectations_cloud-20250904.0.dev1.dist-info}/WHEEL +0 -0
- {great_expectations_cloud-20250903.0.dev0.dist-info → great_expectations_cloud-20250904.0.dev1.dist-info}/entry_points.txt +0 -0
@@ -7,10 +7,13 @@ from sqlalchemy import inspect
|
|
7
7
|
|
8
8
|
if TYPE_CHECKING:
|
9
9
|
from sqlalchemy.engine import Inspector
|
10
|
+
from sqlalchemy.sql.compiler import IdentifierPreparer
|
10
11
|
|
11
12
|
|
12
13
|
def get_asset_names(datasource: SQLDatasource) -> list[str]:
|
13
14
|
inspector: Inspector = inspect(datasource.get_engine())
|
15
|
+
identifier_preparer: IdentifierPreparer = inspector.dialect.identifier_preparer
|
16
|
+
|
14
17
|
if isinstance(datasource, SnowflakeDatasource) and datasource.schema_:
|
15
18
|
# Snowflake-SQLAlchemy uses the default_schema if no schema is provided to get_table_names
|
16
19
|
# Or if the role does not have access to the schema (it silently fails and defaults to using default_schema)
|
@@ -19,8 +22,14 @@ def get_asset_names(datasource: SQLDatasource) -> list[str]:
|
|
19
22
|
# Also converting to list to ensure JSON serializable
|
20
23
|
tables = list(inspector.get_table_names(schema=datasource.schema_))
|
21
24
|
views = list(inspector.get_view_names(schema=datasource.schema_))
|
22
|
-
|
25
|
+
asset_names = tables + views
|
26
|
+
else:
|
27
|
+
tables = list(inspector.get_table_names())
|
28
|
+
views = list(inspector.get_view_names())
|
29
|
+
asset_names = tables + views
|
23
30
|
|
24
|
-
|
25
|
-
|
26
|
-
|
31
|
+
# the identifier preparer adds quotes when they are necessary
|
32
|
+
quoted_asset_names: list[str] = [
|
33
|
+
identifier_preparer.quote(asset_name) for asset_name in asset_names
|
34
|
+
]
|
35
|
+
return quoted_asset_names
|
@@ -16,8 +16,6 @@ from uuid import UUID
|
|
16
16
|
|
17
17
|
import orjson
|
18
18
|
import requests
|
19
|
-
from great_expectations import __version__
|
20
|
-
from great_expectations.core import http
|
21
19
|
from great_expectations.core.http import create_session
|
22
20
|
from great_expectations.data_context.cloud_constants import CLOUD_DEFAULT_BASE_URL
|
23
21
|
from great_expectations.data_context.data_context.context_factory import get_context
|
@@ -152,7 +150,19 @@ class GXAgent:
|
|
152
150
|
"great_expectations_version": great_expectations_version,
|
153
151
|
},
|
154
152
|
)
|
155
|
-
|
153
|
+
LOGGER.debug("Loading a DataContext - this might take a moment.")
|
154
|
+
|
155
|
+
with warnings.catch_warnings():
|
156
|
+
# suppress warnings about GX version
|
157
|
+
warnings.filterwarnings("ignore", message="You are using great_expectations version")
|
158
|
+
self._context: CloudDataContext = get_context(
|
159
|
+
cloud_mode=True,
|
160
|
+
user_agent_str=self.user_agent_str,
|
161
|
+
)
|
162
|
+
self._configure_progress_bars(data_context=self._context)
|
163
|
+
LOGGER.debug("DataContext is ready.")
|
164
|
+
|
165
|
+
self._set_http_session_headers(data_context=self._context)
|
156
166
|
|
157
167
|
# Create a thread pool with a single worker, so we can run long-lived
|
158
168
|
# GX processes and maintain our connection to the broker. Note that
|
@@ -253,7 +263,6 @@ class GXAgent:
|
|
253
263
|
"event_type": event_context.event.type,
|
254
264
|
"correlation_id": event_context.correlation_id,
|
255
265
|
"organization_id": self.get_organization_id(event_context),
|
256
|
-
"workspace_id": str(self.get_workspace_id(event_context)),
|
257
266
|
"schedule_id": event_context.event.schedule_id
|
258
267
|
if isinstance(event_context.event, ScheduledEventBase)
|
259
268
|
else None,
|
@@ -278,19 +287,8 @@ class GXAgent:
|
|
278
287
|
self._current_task.add_done_callback(on_exit_callback)
|
279
288
|
|
280
289
|
def get_data_context(self, event_context: EventContext) -> CloudDataContext:
|
281
|
-
"""
|
282
|
-
|
283
|
-
warnings.filterwarnings("ignore", message="You are using great_expectations version")
|
284
|
-
# Validate presence of workspace_id (required), even if not passed directly to get_context
|
285
|
-
if getattr(event_context.event, "workspace_id", None) is None:
|
286
|
-
raise GXAgentError()
|
287
|
-
|
288
|
-
context: CloudDataContext = get_context(
|
289
|
-
cloud_mode=True,
|
290
|
-
user_agent_str=self.user_agent_str,
|
291
|
-
)
|
292
|
-
self._configure_progress_bars(data_context=context)
|
293
|
-
return context
|
290
|
+
"""Helper method to get a DataContext Agent. Overridden in GX-Runner."""
|
291
|
+
return self._context
|
294
292
|
|
295
293
|
def get_organization_id(self, event_context: EventContext) -> UUID:
|
296
294
|
"""Helper method to get the organization ID. Overridden in GX-Runner."""
|
@@ -300,13 +298,6 @@ class GXAgent:
|
|
300
298
|
"""Helper method to get the auth key. Overridden in GX-Runner."""
|
301
299
|
return self._get_config().gx_cloud_access_token
|
302
300
|
|
303
|
-
def get_workspace_id(self, event_context: EventContext) -> UUID:
|
304
|
-
"""Helper method to get the workspace ID from the event."""
|
305
|
-
workspace_id: UUID | None = getattr(event_context.event, "workspace_id", None)
|
306
|
-
if workspace_id is None:
|
307
|
-
raise GXAgentError()
|
308
|
-
return workspace_id
|
309
|
-
|
310
301
|
def _set_sentry_tags(self, even_context: EventContext) -> None:
|
311
302
|
"""Used by GX-Runner to set tags for Sentry logging. No-op in the Agent."""
|
312
303
|
pass
|
@@ -329,18 +320,14 @@ class GXAgent:
|
|
329
320
|
)
|
330
321
|
|
331
322
|
org_id = self.get_organization_id(event_context)
|
332
|
-
workspace_id = self.get_workspace_id(event_context)
|
333
323
|
base_url = self._get_config().gx_cloud_base_url
|
334
324
|
auth_key = self.get_auth_key()
|
335
325
|
|
336
326
|
if isinstance(event_context.event, ScheduledEventBase):
|
337
|
-
self._create_scheduled_job_and_set_started(event_context, org_id
|
327
|
+
self._create_scheduled_job_and_set_started(event_context, org_id)
|
338
328
|
else:
|
339
329
|
self._update_status(
|
340
|
-
correlation_id=event_context.correlation_id,
|
341
|
-
status=JobStarted(),
|
342
|
-
org_id=org_id,
|
343
|
-
workspace_id=workspace_id,
|
330
|
+
correlation_id=event_context.correlation_id, status=JobStarted(), org_id=org_id
|
344
331
|
)
|
345
332
|
LOGGER.info(
|
346
333
|
"Starting job",
|
@@ -348,7 +335,6 @@ class GXAgent:
|
|
348
335
|
"event_type": event_context.event.type,
|
349
336
|
"correlation_id": event_context.correlation_id,
|
350
337
|
"organization_id": str(org_id),
|
351
|
-
"workspace_id": str(workspace_id),
|
352
338
|
"schedule_id": event_context.event.schedule_id
|
353
339
|
if isinstance(event_context.event, ScheduledEventBase)
|
354
340
|
else None,
|
@@ -380,7 +366,6 @@ class GXAgent:
|
|
380
366
|
# warning: this method will not be executed in the main thread
|
381
367
|
|
382
368
|
org_id = self.get_organization_id(event_context)
|
383
|
-
workspace_id = self.get_workspace_id(event_context)
|
384
369
|
|
385
370
|
# get results or errors from the thread
|
386
371
|
error = future.exception()
|
@@ -400,7 +385,6 @@ class GXAgent:
|
|
400
385
|
"event_type": event_context.event.type,
|
401
386
|
"id": event_context.correlation_id,
|
402
387
|
"organization_id": str(org_id),
|
403
|
-
"workspace_id": str(workspace_id),
|
404
388
|
"schedule_id": event_context.event.schedule_id
|
405
389
|
if isinstance(event_context.event, ScheduledEventBase)
|
406
390
|
else None,
|
@@ -421,7 +405,6 @@ class GXAgent:
|
|
421
405
|
result.job_duration.total_seconds() if result.job_duration else None
|
422
406
|
),
|
423
407
|
"organization_id": str(org_id),
|
424
|
-
"workspace_id": str(workspace_id),
|
425
408
|
"schedule_id": event_context.event.schedule_id
|
426
409
|
if isinstance(event_context.event, ScheduledEventBase)
|
427
410
|
else None,
|
@@ -436,16 +419,12 @@ class GXAgent:
|
|
436
419
|
"event_type": event_context.event.type,
|
437
420
|
"correlation_id": event_context.correlation_id,
|
438
421
|
"organization_id": str(org_id),
|
439
|
-
"workspace_id": str(workspace_id),
|
440
422
|
},
|
441
423
|
)
|
442
424
|
|
443
425
|
try:
|
444
426
|
self._update_status(
|
445
|
-
correlation_id=event_context.correlation_id,
|
446
|
-
status=status,
|
447
|
-
org_id=org_id,
|
448
|
-
workspace_id=workspace_id,
|
427
|
+
correlation_id=event_context.correlation_id, status=status, org_id=org_id
|
449
428
|
)
|
450
429
|
except Exception:
|
451
430
|
LOGGER.exception(
|
@@ -454,7 +433,6 @@ class GXAgent:
|
|
454
433
|
"correlation_id": event_context.correlation_id,
|
455
434
|
"status": str(status),
|
456
435
|
"organization_id": str(org_id),
|
457
|
-
"workspace_id": str(workspace_id),
|
458
436
|
},
|
459
437
|
)
|
460
438
|
# We do not want to cause an infinite loop of errors
|
@@ -574,9 +552,7 @@ class GXAgent:
|
|
574
552
|
)
|
575
553
|
)
|
576
554
|
|
577
|
-
def _update_status(
|
578
|
-
self, correlation_id: str, status: JobStatus, org_id: UUID, workspace_id: UUID
|
579
|
-
) -> None:
|
555
|
+
def _update_status(self, correlation_id: str, status: JobStatus, org_id: UUID) -> None:
|
580
556
|
"""Update GX Cloud on the status of a job.
|
581
557
|
|
582
558
|
Args:
|
@@ -589,12 +565,11 @@ class GXAgent:
|
|
589
565
|
"correlation_id": correlation_id,
|
590
566
|
"status": str(status),
|
591
567
|
"organization_id": str(org_id),
|
592
|
-
"workspace_id": str(workspace_id),
|
593
568
|
},
|
594
569
|
)
|
595
570
|
agent_sessions_url = urljoin(
|
596
571
|
self._get_config().gx_cloud_base_url,
|
597
|
-
f"/api/v1/organizations/{org_id}/
|
572
|
+
f"/api/v1/organizations/{org_id}/agent-jobs/{correlation_id}",
|
598
573
|
)
|
599
574
|
with create_session(access_token=self.get_auth_key()) as session:
|
600
575
|
data = UpdateJobStatusRequest(data=status).json()
|
@@ -605,7 +580,6 @@ class GXAgent:
|
|
605
580
|
"correlation_id": correlation_id,
|
606
581
|
"status": str(status),
|
607
582
|
"organization_id": str(org_id),
|
608
|
-
"workspace_id": str(workspace_id),
|
609
583
|
},
|
610
584
|
)
|
611
585
|
GXAgent._log_http_error(
|
@@ -613,7 +587,7 @@ class GXAgent:
|
|
613
587
|
)
|
614
588
|
|
615
589
|
def _create_scheduled_job_and_set_started(
|
616
|
-
self, event_context: EventContext, org_id: UUID
|
590
|
+
self, event_context: EventContext, org_id: UUID
|
617
591
|
) -> None:
|
618
592
|
"""Create a job in GX Cloud for scheduled events.
|
619
593
|
|
@@ -635,14 +609,13 @@ class GXAgent:
|
|
635
609
|
"correlation_id": str(event_context.correlation_id),
|
636
610
|
"event_type": str(event_context.event.type),
|
637
611
|
"organization_id": str(org_id),
|
638
|
-
"workspace_id": str(workspace_id),
|
639
612
|
"schedule_id": str(event_context.event.schedule_id),
|
640
613
|
},
|
641
614
|
)
|
642
615
|
|
643
616
|
agent_sessions_url = urljoin(
|
644
617
|
self._get_config().gx_cloud_base_url,
|
645
|
-
f"/api/v1/organizations/{org_id}/
|
618
|
+
f"/api/v1/organizations/{org_id}/agent-jobs",
|
646
619
|
)
|
647
620
|
data = CreateScheduledJobAndSetJobStarted(
|
648
621
|
type="run_scheduled_checkpoint.received",
|
@@ -663,7 +636,6 @@ class GXAgent:
|
|
663
636
|
"event_type": str(event_context.event.type),
|
664
637
|
"organization_id": str(org_id),
|
665
638
|
"schedule_id": str(event_context.event.schedule_id),
|
666
|
-
"workspace_id": str(workspace_id),
|
667
639
|
},
|
668
640
|
)
|
669
641
|
GXAgent._log_http_error(
|
@@ -714,6 +686,8 @@ class GXAgent:
|
|
714
686
|
Note: the Agent-Job-Id header value will be set for all GX Cloud request until this method is
|
715
687
|
called again.
|
716
688
|
"""
|
689
|
+
from great_expectations import __version__ # noqa: PLC0415
|
690
|
+
from great_expectations.core import http # noqa: PLC0415
|
717
691
|
|
718
692
|
header_name = self.get_header_name()
|
719
693
|
user_agent_header_value = self.user_agent_str
|
@@ -10,8 +10,8 @@ great_expectations_cloud/agent/actions/run_metric_list_action.py,sha256=tW64pNYJ
|
|
10
10
|
great_expectations_cloud/agent/actions/run_scheduled_checkpoint.py,sha256=XMvv59izhzjSuw2YF7mlyNkeUQ3hxXj6G-ldAeSfl-s,2403
|
11
11
|
great_expectations_cloud/agent/actions/run_window_checkpoint.py,sha256=khFmTSZ6W-MrInswgEHGrQAzVt-aelycurAMPbcLcLU,2264
|
12
12
|
great_expectations_cloud/agent/actions/unknown.py,sha256=mtWw9tDZqGZSiUWj7PtIlLFJ1dM-7AHBX3SO16-u2EM,739
|
13
|
-
great_expectations_cloud/agent/actions/utils.py,sha256=
|
14
|
-
great_expectations_cloud/agent/agent.py,sha256=
|
13
|
+
great_expectations_cloud/agent/actions/utils.py,sha256=0lzeASN1TYWJ35-H-sCRkcPMHzWU05SzKIyTv0CzvA8,1665
|
14
|
+
great_expectations_cloud/agent/agent.py,sha256=Gd8gaamnh7Gw4uiW2-sVxbMaWdeNibHQR6LDryI_uVY,29761
|
15
15
|
great_expectations_cloud/agent/agent_warnings.py,sha256=9-xl_AI2V9Py4o7KzFOQjG3lYx-vZ36fq4w2iiPNiUw,362
|
16
16
|
great_expectations_cloud/agent/cli.py,sha256=a_HmPxBMlVD59BEmkZnlbOOAFlezVMx9djZ2XIW-3W0,2885
|
17
17
|
great_expectations_cloud/agent/config.py,sha256=c1BOr-TrxZnciWNRuu4atGtfRh-XSmwIS0osDCzQa04,1348
|
@@ -21,14 +21,14 @@ great_expectations_cloud/agent/exceptions.py,sha256=XIDBVSmBfFpVQA5i9rZqEtORIdi5
|
|
21
21
|
great_expectations_cloud/agent/message_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
22
|
great_expectations_cloud/agent/message_service/asyncio_rabbit_mq_client.py,sha256=B2EwgG_Qdm1s_xkxbGPQxdRBkDSuhkKyT6rGFpXuqQ0,12391
|
23
23
|
great_expectations_cloud/agent/message_service/subscriber.py,sha256=K8szy9uM1MUBkMaMZt9o5ExnjqEn1hw6zYaH7pQkjoM,5975
|
24
|
-
great_expectations_cloud/agent/models.py,sha256=
|
24
|
+
great_expectations_cloud/agent/models.py,sha256=yZEir-ZShLbyJ8DZxMYNAZmSzbpOSH5wGRKyPo1LcJU,9984
|
25
25
|
great_expectations_cloud/agent/run.py,sha256=V33RLoB1PFmJ0h0RfHG4SB5lN_Za8tW2Dua6GUpN9yY,639
|
26
26
|
great_expectations_cloud/agent/utils.py,sha256=3OvdcXeK1gk2oJgqG4jPvBRwlMCn8LioULW3YgRtj98,2950
|
27
27
|
great_expectations_cloud/logging/README.md,sha256=vbwU689x8SkGjzoBYQzZOzAvh28fR0RCa1XY5WD-Dgs,1762
|
28
28
|
great_expectations_cloud/logging/logging_cfg.py,sha256=W6mlm4_Z2bjzM5TuKmFg_WZor2XoJm4DAoLGaf2O__I,6579
|
29
29
|
great_expectations_cloud/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
-
great_expectations_cloud-
|
31
|
-
great_expectations_cloud-
|
32
|
-
great_expectations_cloud-
|
33
|
-
great_expectations_cloud-
|
34
|
-
great_expectations_cloud-
|
30
|
+
great_expectations_cloud-20250904.0.dev1.dist-info/LICENSE,sha256=_JJnoX6N_OkrAwlCRizCwil0tIjDAy2TG3GiJ50sM6k,2084
|
31
|
+
great_expectations_cloud-20250904.0.dev1.dist-info/METADATA,sha256=zzuWd4us-v3-yfX1zq5W6SYWRzNmPNnmVOqlJJp8xII,12352
|
32
|
+
great_expectations_cloud-20250904.0.dev1.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
|
33
|
+
great_expectations_cloud-20250904.0.dev1.dist-info/entry_points.txt,sha256=ofJgdeS2gSzxXLyCAjfNhIaN1wmSyR7EAMs5qhVaXE4,68
|
34
|
+
great_expectations_cloud-20250904.0.dev1.dist-info/RECORD,,
|
File without changes
|
File without changes
|