garf-executors 0.2.3__py3-none-any.whl → 1.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- garf/executors/__init__.py +25 -0
- garf/executors/api_executor.py +228 -0
- garf/executors/bq_executor.py +179 -0
- garf/executors/config.py +52 -0
- garf/executors/entrypoints/__init__.py +0 -0
- garf/executors/entrypoints/cli.py +164 -0
- {garf_executors → garf/executors}/entrypoints/grpc_server.py +22 -9
- garf/executors/entrypoints/server.py +174 -0
- garf/executors/entrypoints/tracer.py +82 -0
- garf/executors/entrypoints/utils.py +140 -0
- garf/executors/exceptions.py +17 -0
- garf/executors/execution_context.py +117 -0
- garf/executors/executor.py +124 -0
- garf/executors/fetchers.py +128 -0
- garf/executors/garf_pb2.py +51 -0
- {garf_executors → garf/executors}/garf_pb2_grpc.py +45 -2
- garf/executors/query_processor.py +79 -0
- garf/executors/setup.py +58 -0
- garf/executors/sql_executor.py +144 -0
- garf/executors/telemetry.py +20 -0
- garf/executors/workflows/__init__.py +0 -0
- garf/executors/workflows/gcp_workflow.yaml +49 -0
- garf/executors/workflows/workflow.py +164 -0
- garf/executors/workflows/workflow_runner.py +172 -0
- garf_executors/__init__.py +9 -44
- garf_executors/api_executor.py +9 -121
- garf_executors/bq_executor.py +9 -161
- garf_executors/config.py +9 -37
- garf_executors/entrypoints/__init__.py +25 -0
- garf_executors/entrypoints/cli.py +9 -148
- garf_executors/entrypoints/grcp_server.py +25 -0
- garf_executors/entrypoints/server.py +9 -102
- garf_executors/entrypoints/tracer.py +8 -40
- garf_executors/entrypoints/utils.py +9 -124
- garf_executors/exceptions.py +11 -3
- garf_executors/execution_context.py +9 -100
- garf_executors/executor.py +9 -108
- garf_executors/fetchers.py +9 -63
- garf_executors/sql_executor.py +9 -125
- garf_executors/telemetry.py +10 -5
- garf_executors/workflow.py +8 -79
- {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/METADATA +18 -5
- garf_executors-1.1.3.dist-info/RECORD +46 -0
- {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/WHEEL +1 -1
- garf_executors-1.1.3.dist-info/entry_points.txt +2 -0
- {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/top_level.txt +1 -0
- garf_executors/garf_pb2.py +0 -45
- garf_executors-0.2.3.dist-info/RECORD +0 -24
- garf_executors-0.2.3.dist-info/entry_points.txt +0 -2
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Copyright 2025 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Defines common functionality between executors."""
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import inspect
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
from garf.core import report_fetcher
|
|
22
|
+
from garf.executors import execution_context, query_processor
|
|
23
|
+
from garf.executors.telemetry import tracer
|
|
24
|
+
from opentelemetry import trace
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Executor:
|
|
28
|
+
"""Defines common functionality between executors."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
preprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
|
|
33
|
+
postprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
self.preprocessors = preprocessors or {}
|
|
36
|
+
self.postprocessors = postprocessors or {}
|
|
37
|
+
|
|
38
|
+
@tracer.start_as_current_span('api.execute_batch')
|
|
39
|
+
def execute_batch(
|
|
40
|
+
self,
|
|
41
|
+
batch: dict[str, str],
|
|
42
|
+
context: execution_context.ExecutionContext,
|
|
43
|
+
parallel_threshold: int = 10,
|
|
44
|
+
) -> list[str]:
|
|
45
|
+
"""Executes batch of queries for a common context.
|
|
46
|
+
|
|
47
|
+
If an executor has any pre/post processors, executes them first while
|
|
48
|
+
modifying the context.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
batch: Mapping between query_title and its text.
|
|
52
|
+
context: Execution context.
|
|
53
|
+
parallel_threshold: Number of queries to execute in parallel.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
Results of execution.
|
|
57
|
+
"""
|
|
58
|
+
span = trace.get_current_span()
|
|
59
|
+
span.set_attribute('api.parallel_threshold', parallel_threshold)
|
|
60
|
+
_handle_processors(processors=self.preprocessors, context=context)
|
|
61
|
+
results = asyncio.run(
|
|
62
|
+
self._run(
|
|
63
|
+
batch=batch, context=context, parallel_threshold=parallel_threshold
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
_handle_processors(processors=self.postprocessors, context=context)
|
|
67
|
+
return results
|
|
68
|
+
|
|
69
|
+
def add_preprocessor(
|
|
70
|
+
self, preprocessors: dict[str, report_fetcher.Processor]
|
|
71
|
+
) -> None:
|
|
72
|
+
self.preprocessors.update(preprocessors)
|
|
73
|
+
|
|
74
|
+
async def aexecute(
|
|
75
|
+
self,
|
|
76
|
+
query: str,
|
|
77
|
+
title: str,
|
|
78
|
+
context: execution_context.ExecutionContext,
|
|
79
|
+
) -> str:
|
|
80
|
+
"""Performs query execution asynchronously.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
query: Location of the query.
|
|
84
|
+
title: Name of the query.
|
|
85
|
+
context: Query execution context.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Result of writing the report.
|
|
89
|
+
"""
|
|
90
|
+
return await asyncio.to_thread(self.execute, query, title, context)
|
|
91
|
+
|
|
92
|
+
async def _run(
|
|
93
|
+
self,
|
|
94
|
+
batch: dict[str, str],
|
|
95
|
+
context: execution_context.ExecutionContext,
|
|
96
|
+
parallel_threshold: int,
|
|
97
|
+
):
|
|
98
|
+
semaphore = asyncio.Semaphore(value=parallel_threshold)
|
|
99
|
+
|
|
100
|
+
async def run_with_semaphore(fn):
|
|
101
|
+
async with semaphore:
|
|
102
|
+
return await fn
|
|
103
|
+
|
|
104
|
+
tasks = [
|
|
105
|
+
self.aexecute(query=query, title=title, context=context)
|
|
106
|
+
for title, query in batch.items()
|
|
107
|
+
]
|
|
108
|
+
return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _handle_processors(
|
|
112
|
+
processors: dict[str, report_fetcher.Processor],
|
|
113
|
+
context: execution_context.ExecutionContext,
|
|
114
|
+
) -> None:
|
|
115
|
+
context = query_processor.process_gquery(context)
|
|
116
|
+
for k, processor in processors.items():
|
|
117
|
+
processor_signature = list(inspect.signature(processor).parameters.keys())
|
|
118
|
+
if k in context.fetcher_parameters:
|
|
119
|
+
processor_parameters = {
|
|
120
|
+
k: v
|
|
121
|
+
for k, v in context.fetcher_parameters.items()
|
|
122
|
+
if k in processor_signature
|
|
123
|
+
}
|
|
124
|
+
context.fetcher_parameters[k] = processor(**processor_parameters)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Copyright 2025 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import inspect
|
|
16
|
+
import logging
|
|
17
|
+
import sys
|
|
18
|
+
from importlib.metadata import entry_points
|
|
19
|
+
|
|
20
|
+
from garf.core import report_fetcher, simulator
|
|
21
|
+
from garf.executors.telemetry import tracer
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(name='garf.executors.fetchers')
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@tracer.start_as_current_span('find_fetchers')
|
|
27
|
+
def find_fetchers() -> set[str]:
|
|
28
|
+
"""Identifiers all available report fetchers."""
|
|
29
|
+
if entrypoints := _get_entrypoints('garf'):
|
|
30
|
+
return {fetcher.name for fetcher in entrypoints}
|
|
31
|
+
return set()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@tracer.start_as_current_span('find_simulators')
|
|
35
|
+
def find_simulators() -> set[str]:
|
|
36
|
+
"""Identifiers all available report simulators."""
|
|
37
|
+
if entrypoints := _get_entrypoints('garf_simulator'):
|
|
38
|
+
return {simulator.name for simulator in entrypoints}
|
|
39
|
+
return set()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@tracer.start_as_current_span('get_report_fetcher')
|
|
43
|
+
def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
|
|
44
|
+
"""Loads report fetcher for a given source.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
source: Alias for a source associated with a fetcher.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
Class for a found report fetcher.
|
|
51
|
+
|
|
52
|
+
Raises:
|
|
53
|
+
ApiReportFetcherError: When fetcher cannot be loaded.
|
|
54
|
+
MissingApiReportFetcherError: When fetcher not found.
|
|
55
|
+
"""
|
|
56
|
+
if source not in find_fetchers():
|
|
57
|
+
raise report_fetcher.MissingApiReportFetcherError(source)
|
|
58
|
+
for fetcher in _get_entrypoints('garf'):
|
|
59
|
+
if fetcher.name == source:
|
|
60
|
+
try:
|
|
61
|
+
with tracer.start_as_current_span('load_fetcher_module') as span:
|
|
62
|
+
fetcher_module = fetcher.load()
|
|
63
|
+
span.set_attribute('loaded_module', fetcher_module.__name__)
|
|
64
|
+
for name, obj in inspect.getmembers(fetcher_module):
|
|
65
|
+
if inspect.isclass(obj) and issubclass(
|
|
66
|
+
obj, report_fetcher.ApiReportFetcher
|
|
67
|
+
):
|
|
68
|
+
if not hasattr(obj, 'alias'):
|
|
69
|
+
return getattr(fetcher_module, name)
|
|
70
|
+
if obj.alias == fetcher.name:
|
|
71
|
+
return getattr(fetcher_module, name)
|
|
72
|
+
except ModuleNotFoundError as e:
|
|
73
|
+
raise report_fetcher.ApiReportFetcherError(
|
|
74
|
+
f'Failed to load fetcher for source {source}, reason: {e}'
|
|
75
|
+
)
|
|
76
|
+
raise report_fetcher.ApiReportFetcherError(
|
|
77
|
+
f'No fetcher available for the source "{source}"'
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@tracer.start_as_current_span('get_report_simulator')
|
|
82
|
+
def get_report_simulator(source: str) -> type[simulator.ApiReportSimulator]:
|
|
83
|
+
"""Loads report simulator for a given source.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
source: Alias for a source associated with a simulator.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
Class for a found report simulator.
|
|
90
|
+
|
|
91
|
+
Raises:
|
|
92
|
+
GarfApiReportSimulatorError: When simulator cannot be loaded.
|
|
93
|
+
MissingApiReportSimulatorError: When simulator not found.
|
|
94
|
+
"""
|
|
95
|
+
if source not in find_simulators():
|
|
96
|
+
raise simulator.MissingApiReportSimulatorError(source)
|
|
97
|
+
for sim in _get_entrypoints('garf_simulator'):
|
|
98
|
+
if sim.name == source:
|
|
99
|
+
try:
|
|
100
|
+
with tracer.start_as_current_span('load_simulator_module') as span:
|
|
101
|
+
simulator_module = sim.load()
|
|
102
|
+
span.set_attribute('loaded_module', simulator_module.__name__)
|
|
103
|
+
for name, obj in inspect.getmembers(simulator_module):
|
|
104
|
+
if inspect.isclass(obj) and issubclass(
|
|
105
|
+
obj, simulator.ApiReportSimulator
|
|
106
|
+
):
|
|
107
|
+
if not hasattr(obj, 'alias'):
|
|
108
|
+
return getattr(simulator_module, name)
|
|
109
|
+
if obj.alias == sim.name:
|
|
110
|
+
return getattr(simulator_module, name)
|
|
111
|
+
except ModuleNotFoundError as e:
|
|
112
|
+
raise simulator.GarfApiReportSimulatorError(
|
|
113
|
+
f'Failed to load simulator for source {source}, reason: {e}'
|
|
114
|
+
)
|
|
115
|
+
raise simulator.GarfApiReportSimulatorError(
|
|
116
|
+
f'No simulator available for the source "{source}"'
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _get_entrypoints(group='garf'):
|
|
121
|
+
if sys.version_info.major == 3 and sys.version_info.minor == 9:
|
|
122
|
+
try:
|
|
123
|
+
fetchers = entry_points()[group]
|
|
124
|
+
except KeyError:
|
|
125
|
+
fetchers = []
|
|
126
|
+
else:
|
|
127
|
+
fetchers = entry_points(group=group)
|
|
128
|
+
return fetchers
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# NO CHECKED-IN PROTOBUF GENCODE
|
|
4
|
+
# source: garf.proto
|
|
5
|
+
# Protobuf Python Version: 6.31.1
|
|
6
|
+
"""Generated protocol buffer code."""
|
|
7
|
+
from google.protobuf import descriptor as _descriptor
|
|
8
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
9
|
+
from google.protobuf import runtime_version as _runtime_version
|
|
10
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
11
|
+
from google.protobuf.internal import builder as _builder
|
|
12
|
+
_runtime_version.ValidateProtobufRuntimeVersion(
|
|
13
|
+
_runtime_version.Domain.PUBLIC,
|
|
14
|
+
6,
|
|
15
|
+
31,
|
|
16
|
+
1,
|
|
17
|
+
'',
|
|
18
|
+
'garf.proto'
|
|
19
|
+
)
|
|
20
|
+
# @@protoc_insertion_point(imports)
|
|
21
|
+
|
|
22
|
+
_sym_db = _symbol_database.Default()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\ngarf.proto\x12\x04garf\x1a\x1cgoogle/protobuf/struct.proto\"a\n\x0c\x46\x65tchRequest\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12\r\n\x05query\x18\x03 \x01(\t\x12#\n\x07\x63ontext\x18\x04 \x01(\x0b\x32\x12.garf.FetchContext\"G\n\rFetchResponse\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\x12%\n\x04rows\x18\x02 \x03(\x0b\x32\x17.google.protobuf.Struct\"t\n\x0c\x46\x65tchContext\x12/\n\x10query_parameters\x18\x01 \x01(\x0b\x32\x15.garf.QueryParameters\x12\x33\n\x12\x66\x65tcher_parameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"g\n\x0e\x45xecuteRequest\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12\r\n\x05query\x18\x03 \x01(\t\x12\'\n\x07\x63ontext\x18\x04 \x01(\x0b\x32\x16.garf.ExecutionContext\"\xbc\x01\n\x10\x45xecutionContext\x12/\n\x10query_parameters\x18\x01 \x01(\x0b\x32\x15.garf.QueryParameters\x12\x33\n\x12\x66\x65tcher_parameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x0e\n\x06writer\x18\x03 \x01(\t\x12\x32\n\x11writer_parameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\"d\n\x0fQueryParameters\x12&\n\x05macro\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\x12)\n\x08template\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"\"\n\x0f\x45xecuteResponse\x12\x0f\n\x07results\x18\x01 \x03(\t2{\n\x0bGarfService\x12\x38\n\x07\x45xecute\x12\x14.garf.ExecuteRequest\x1a\x15.garf.ExecuteResponse\"\x00\x12\x32\n\x05\x46\x65tch\x12\x12.garf.FetchRequest\x1a\x13.garf.FetchResponse\"\x00\x62\x06proto3')
|
|
29
|
+
|
|
30
|
+
_globals = globals()
|
|
31
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
32
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'garf_pb2', _globals)
|
|
33
|
+
if not _descriptor._USE_C_DESCRIPTORS:
|
|
34
|
+
DESCRIPTOR._loaded_options = None
|
|
35
|
+
_globals['_FETCHREQUEST']._serialized_start=50
|
|
36
|
+
_globals['_FETCHREQUEST']._serialized_end=147
|
|
37
|
+
_globals['_FETCHRESPONSE']._serialized_start=149
|
|
38
|
+
_globals['_FETCHRESPONSE']._serialized_end=220
|
|
39
|
+
_globals['_FETCHCONTEXT']._serialized_start=222
|
|
40
|
+
_globals['_FETCHCONTEXT']._serialized_end=338
|
|
41
|
+
_globals['_EXECUTEREQUEST']._serialized_start=340
|
|
42
|
+
_globals['_EXECUTEREQUEST']._serialized_end=443
|
|
43
|
+
_globals['_EXECUTIONCONTEXT']._serialized_start=446
|
|
44
|
+
_globals['_EXECUTIONCONTEXT']._serialized_end=634
|
|
45
|
+
_globals['_QUERYPARAMETERS']._serialized_start=636
|
|
46
|
+
_globals['_QUERYPARAMETERS']._serialized_end=736
|
|
47
|
+
_globals['_EXECUTERESPONSE']._serialized_start=738
|
|
48
|
+
_globals['_EXECUTERESPONSE']._serialized_end=772
|
|
49
|
+
_globals['_GARFSERVICE']._serialized_start=774
|
|
50
|
+
_globals['_GARFSERVICE']._serialized_end=897
|
|
51
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -5,7 +5,7 @@ import warnings
|
|
|
5
5
|
|
|
6
6
|
from . import garf_pb2 as garf__pb2
|
|
7
7
|
|
|
8
|
-
GRPC_GENERATED_VERSION = '1.
|
|
8
|
+
GRPC_GENERATED_VERSION = '1.76.0'
|
|
9
9
|
GRPC_VERSION = grpc.__version__
|
|
10
10
|
_version_not_supported = False
|
|
11
11
|
|
|
@@ -18,7 +18,7 @@ except ImportError:
|
|
|
18
18
|
if _version_not_supported:
|
|
19
19
|
raise RuntimeError(
|
|
20
20
|
f'The grpc package installed is at version {GRPC_VERSION},'
|
|
21
|
-
+
|
|
21
|
+
+ ' but the generated code in garf_pb2_grpc.py depends on'
|
|
22
22
|
+ f' grpcio>={GRPC_GENERATED_VERSION}.'
|
|
23
23
|
+ f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
|
|
24
24
|
+ f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
|
|
@@ -39,6 +39,11 @@ class GarfServiceStub(object):
|
|
|
39
39
|
request_serializer=garf__pb2.ExecuteRequest.SerializeToString,
|
|
40
40
|
response_deserializer=garf__pb2.ExecuteResponse.FromString,
|
|
41
41
|
_registered_method=True)
|
|
42
|
+
self.Fetch = channel.unary_unary(
|
|
43
|
+
'/garf.GarfService/Fetch',
|
|
44
|
+
request_serializer=garf__pb2.FetchRequest.SerializeToString,
|
|
45
|
+
response_deserializer=garf__pb2.FetchResponse.FromString,
|
|
46
|
+
_registered_method=True)
|
|
42
47
|
|
|
43
48
|
|
|
44
49
|
class GarfServiceServicer(object):
|
|
@@ -50,6 +55,12 @@ class GarfServiceServicer(object):
|
|
|
50
55
|
context.set_details('Method not implemented!')
|
|
51
56
|
raise NotImplementedError('Method not implemented!')
|
|
52
57
|
|
|
58
|
+
def Fetch(self, request, context):
|
|
59
|
+
"""Missing associated documentation comment in .proto file."""
|
|
60
|
+
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
|
61
|
+
context.set_details('Method not implemented!')
|
|
62
|
+
raise NotImplementedError('Method not implemented!')
|
|
63
|
+
|
|
53
64
|
|
|
54
65
|
def add_GarfServiceServicer_to_server(servicer, server):
|
|
55
66
|
rpc_method_handlers = {
|
|
@@ -58,6 +69,11 @@ def add_GarfServiceServicer_to_server(servicer, server):
|
|
|
58
69
|
request_deserializer=garf__pb2.ExecuteRequest.FromString,
|
|
59
70
|
response_serializer=garf__pb2.ExecuteResponse.SerializeToString,
|
|
60
71
|
),
|
|
72
|
+
'Fetch': grpc.unary_unary_rpc_method_handler(
|
|
73
|
+
servicer.Fetch,
|
|
74
|
+
request_deserializer=garf__pb2.FetchRequest.FromString,
|
|
75
|
+
response_serializer=garf__pb2.FetchResponse.SerializeToString,
|
|
76
|
+
),
|
|
61
77
|
}
|
|
62
78
|
generic_handler = grpc.method_handlers_generic_handler(
|
|
63
79
|
'garf.GarfService', rpc_method_handlers)
|
|
@@ -95,3 +111,30 @@ class GarfService(object):
|
|
|
95
111
|
timeout,
|
|
96
112
|
metadata,
|
|
97
113
|
_registered_method=True)
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def Fetch(request,
|
|
117
|
+
target,
|
|
118
|
+
options=(),
|
|
119
|
+
channel_credentials=None,
|
|
120
|
+
call_credentials=None,
|
|
121
|
+
insecure=False,
|
|
122
|
+
compression=None,
|
|
123
|
+
wait_for_ready=None,
|
|
124
|
+
timeout=None,
|
|
125
|
+
metadata=None):
|
|
126
|
+
return grpc.experimental.unary_unary(
|
|
127
|
+
request,
|
|
128
|
+
target,
|
|
129
|
+
'/garf.GarfService/Fetch',
|
|
130
|
+
garf__pb2.FetchRequest.SerializeToString,
|
|
131
|
+
garf__pb2.FetchResponse.FromString,
|
|
132
|
+
options,
|
|
133
|
+
channel_credentials,
|
|
134
|
+
insecure,
|
|
135
|
+
call_credentials,
|
|
136
|
+
compression,
|
|
137
|
+
wait_for_ready,
|
|
138
|
+
timeout,
|
|
139
|
+
metadata,
|
|
140
|
+
_registered_method=True)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""qQuery can be used as a parameter in garf queries."""
|
|
16
|
+
|
|
17
|
+
import contextlib
|
|
18
|
+
|
|
19
|
+
from garf.core import query_editor, query_parser
|
|
20
|
+
from garf.executors import execution_context
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class GqueryError(query_parser.GarfQueryError):
|
|
24
|
+
"""Errors on incorrect qQuery syntax."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _handle_sub_context(context, sub_context):
|
|
28
|
+
for k, v in sub_context.items():
|
|
29
|
+
if isinstance(v, str) and v.startswith('gquery'):
|
|
30
|
+
no_writer_context = context.model_copy(update={'writer': None})
|
|
31
|
+
try:
|
|
32
|
+
_, alias, *query = v.split(':', maxsplit=3)
|
|
33
|
+
except ValueError:
|
|
34
|
+
raise GqueryError(
|
|
35
|
+
f'Incorrect gquery format, should be gquery:alias:query, got {v}'
|
|
36
|
+
)
|
|
37
|
+
if not alias:
|
|
38
|
+
raise GqueryError(f'Missing alias in gquery: {v}')
|
|
39
|
+
if not query:
|
|
40
|
+
raise GqueryError(f'Missing query text in gquery: {v}')
|
|
41
|
+
if alias == 'sqldb':
|
|
42
|
+
from garf.executors import sql_executor
|
|
43
|
+
|
|
44
|
+
gquery_executor = (
|
|
45
|
+
sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
|
|
46
|
+
context.fetcher_parameters.get('connection_string')
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
elif alias == 'bq':
|
|
50
|
+
from garf.executors import bq_executor
|
|
51
|
+
|
|
52
|
+
gquery_executor = bq_executor.BigQueryExecutor(
|
|
53
|
+
**context.fetcher_parameters
|
|
54
|
+
)
|
|
55
|
+
else:
|
|
56
|
+
raise GqueryError(f'Unsupported alias {alias} for gquery: {v}')
|
|
57
|
+
with contextlib.suppress(
|
|
58
|
+
query_editor.GarfResourceError, query_parser.GarfVirtualColumnError
|
|
59
|
+
):
|
|
60
|
+
query = ':'.join(query)
|
|
61
|
+
query_spec = query_editor.QuerySpecification(
|
|
62
|
+
text=query, args=context.query_parameters
|
|
63
|
+
).generate()
|
|
64
|
+
if len(columns := [c for c in query_spec.column_names if c != '_']) > 1:
|
|
65
|
+
raise GqueryError(f'Multiple columns in gquery definition: {columns}')
|
|
66
|
+
res = gquery_executor.execute(
|
|
67
|
+
query=query, title='gquery', context=no_writer_context
|
|
68
|
+
)
|
|
69
|
+
if len(columns := [c for c in res.column_names if c != '_']) > 1:
|
|
70
|
+
raise GqueryError(f'Multiple columns in gquery result: {columns}')
|
|
71
|
+
sub_context[k] = res.to_list(row_type='scalar')
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def process_gquery(
|
|
75
|
+
context: execution_context.ExecutionContext,
|
|
76
|
+
) -> execution_context.ExecutionContext:
|
|
77
|
+
_handle_sub_context(context, context.fetcher_parameters)
|
|
78
|
+
_handle_sub_context(context, context.query_parameters.macro)
|
|
79
|
+
return context
|
garf/executors/setup.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Bootstraps executor based on provided parameters."""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import importlib
|
|
19
|
+
|
|
20
|
+
from garf.executors import executor, fetchers
|
|
21
|
+
from garf.executors.api_executor import ApiQueryExecutor
|
|
22
|
+
from garf.executors.telemetry import tracer
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@tracer.start_as_current_span('setup_executor')
|
|
26
|
+
def setup_executor(
|
|
27
|
+
source: str,
|
|
28
|
+
fetcher_parameters: dict[str, str | int | bool],
|
|
29
|
+
enable_cache: bool = False,
|
|
30
|
+
cache_ttl_seconds: int = 3600,
|
|
31
|
+
simulate: bool = False,
|
|
32
|
+
) -> type[executor.Executor]:
|
|
33
|
+
"""Initializes executors based on a source and parameters."""
|
|
34
|
+
if source == 'bq':
|
|
35
|
+
bq_executor = importlib.import_module('garf.executors.bq_executor')
|
|
36
|
+
query_executor = bq_executor.BigQueryExecutor(**fetcher_parameters)
|
|
37
|
+
elif source == 'sqldb':
|
|
38
|
+
sql_executor = importlib.import_module('garf.executors.sql_executor')
|
|
39
|
+
query_executor = (
|
|
40
|
+
sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
|
|
41
|
+
fetcher_parameters.get('connection_string')
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
else:
|
|
45
|
+
concrete_api_fetcher = fetchers.get_report_fetcher(source)
|
|
46
|
+
if simulate:
|
|
47
|
+
concrete_simulator = fetchers.get_report_simulator(source)()
|
|
48
|
+
else:
|
|
49
|
+
concrete_simulator = None
|
|
50
|
+
query_executor = ApiQueryExecutor(
|
|
51
|
+
fetcher=concrete_api_fetcher(
|
|
52
|
+
**fetcher_parameters,
|
|
53
|
+
enable_cache=enable_cache,
|
|
54
|
+
cache_ttl_seconds=cache_ttl_seconds,
|
|
55
|
+
),
|
|
56
|
+
report_simulator=concrete_simulator,
|
|
57
|
+
)
|
|
58
|
+
return query_executor
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Defines mechanism for executing queries via SqlAlchemy."""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
import sqlalchemy
|
|
20
|
+
except ImportError as e:
|
|
21
|
+
raise ImportError(
|
|
22
|
+
'Please install garf-executors with sqlalchemy support '
|
|
23
|
+
'- `pip install garf-executors[sqlalchemy]`'
|
|
24
|
+
) from e
|
|
25
|
+
|
|
26
|
+
import logging
|
|
27
|
+
import re
|
|
28
|
+
import uuid
|
|
29
|
+
|
|
30
|
+
import pandas as pd
|
|
31
|
+
from garf.core import query_editor, report
|
|
32
|
+
from garf.executors import exceptions, execution_context, executor
|
|
33
|
+
from garf.executors.telemetry import tracer
|
|
34
|
+
from opentelemetry import trace
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SqlAlchemyQueryExecutorError(exceptions.GarfExecutorError):
|
|
40
|
+
"""Error when SqlAlchemyQueryExecutor fails to run query."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SqlAlchemyQueryExecutor(
|
|
44
|
+
executor.Executor, query_editor.TemplateProcessorMixin
|
|
45
|
+
):
|
|
46
|
+
"""Handles query execution via SqlAlchemy.
|
|
47
|
+
|
|
48
|
+
Attributes:
|
|
49
|
+
engine: Initialized Engine object to operated on a given database.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self, engine: sqlalchemy.engine.base.Engine | None = None, **kwargs: str
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Initializes executor with a given engine.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
engine: Initialized Engine object to operated on a given database.
|
|
59
|
+
"""
|
|
60
|
+
self.engine = engine or sqlalchemy.create_engine('sqlite://')
|
|
61
|
+
super().__init__()
|
|
62
|
+
|
|
63
|
+
@classmethod
|
|
64
|
+
def from_connection_string(
|
|
65
|
+
cls, connection_string: str | None
|
|
66
|
+
) -> SqlAlchemyQueryExecutor:
|
|
67
|
+
"""Creates executor from SqlAlchemy connection string.
|
|
68
|
+
|
|
69
|
+
https://docs.sqlalchemy.org/en/20/core/engines.html
|
|
70
|
+
"""
|
|
71
|
+
engine = sqlalchemy.create_engine(connection_string or 'sqlite://')
|
|
72
|
+
return cls(engine)
|
|
73
|
+
|
|
74
|
+
@tracer.start_as_current_span('sql.execute')
|
|
75
|
+
def execute(
|
|
76
|
+
self,
|
|
77
|
+
query: str,
|
|
78
|
+
title: str,
|
|
79
|
+
context: execution_context.ExecutionContext = (
|
|
80
|
+
execution_context.ExecutionContext()
|
|
81
|
+
),
|
|
82
|
+
) -> report.GarfReport:
|
|
83
|
+
"""Executes query in a given database via SqlAlchemy.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
query: Location of the query.
|
|
87
|
+
title: Name of the query.
|
|
88
|
+
context: Query execution context.
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
Report with data if query returns some data otherwise empty Report.
|
|
92
|
+
"""
|
|
93
|
+
span = trace.get_current_span()
|
|
94
|
+
span.set_attribute('query.title', title)
|
|
95
|
+
span.set_attribute('query.text', query)
|
|
96
|
+
logger.info('Executing script: %s', title)
|
|
97
|
+
query_text = self.replace_params_template(query, context.query_parameters)
|
|
98
|
+
with self.engine.begin() as conn:
|
|
99
|
+
if re.findall(r'(create|update) ', query_text.lower()):
|
|
100
|
+
try:
|
|
101
|
+
conn.connection.executescript(query_text)
|
|
102
|
+
results = report.GarfReport()
|
|
103
|
+
except Exception as e:
|
|
104
|
+
raise SqlAlchemyQueryExecutorError(
|
|
105
|
+
f'Failed to execute query {title}: Reason: {e}'
|
|
106
|
+
) from e
|
|
107
|
+
else:
|
|
108
|
+
temp_table_name = f'temp_{uuid.uuid4().hex}'
|
|
109
|
+
query_text = f'CREATE TABLE {temp_table_name} AS {query_text}'
|
|
110
|
+
conn.connection.executescript(query_text)
|
|
111
|
+
try:
|
|
112
|
+
results = report.GarfReport.from_pandas(
|
|
113
|
+
pd.read_sql(f'SELECT * FROM {temp_table_name}', conn)
|
|
114
|
+
)
|
|
115
|
+
except Exception as e:
|
|
116
|
+
raise SqlAlchemyQueryExecutorError(
|
|
117
|
+
f'Failed to execute query {title}: Reason: {e}'
|
|
118
|
+
) from e
|
|
119
|
+
finally:
|
|
120
|
+
conn.connection.execute(f'DROP TABLE {temp_table_name}')
|
|
121
|
+
if context.writer and results:
|
|
122
|
+
writer_clients = context.writer_clients
|
|
123
|
+
if not writer_clients:
|
|
124
|
+
logger.warning('No writers configured, skipping write operation')
|
|
125
|
+
else:
|
|
126
|
+
writing_results = []
|
|
127
|
+
for writer_client in writer_clients:
|
|
128
|
+
logger.debug(
|
|
129
|
+
'Start writing data for query %s via %s writer',
|
|
130
|
+
title,
|
|
131
|
+
type(writer_client),
|
|
132
|
+
)
|
|
133
|
+
writing_result = writer_client.write(results, title)
|
|
134
|
+
logger.debug(
|
|
135
|
+
'Finish writing data for query %s via %s writer',
|
|
136
|
+
title,
|
|
137
|
+
type(writer_client),
|
|
138
|
+
)
|
|
139
|
+
writing_results.append(writing_result)
|
|
140
|
+
logger.info('%s executed successfully', title)
|
|
141
|
+
# Return the last writer's result for backward compatibility
|
|
142
|
+
return writing_results[-1] if writing_results else None
|
|
143
|
+
span.set_attribute('execute.num_results', len(results))
|
|
144
|
+
return results
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Copyright 2025 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
# pylint: disable=C0330, g-bad-import-order, g-multiple-import
|
|
16
|
+
from opentelemetry import trace
|
|
17
|
+
|
|
18
|
+
tracer = trace.get_tracer(
|
|
19
|
+
instrumenting_module_name='garf.executors',
|
|
20
|
+
)
|