garf-executors 0.2.3__py3-none-any.whl → 1.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. garf/executors/__init__.py +25 -0
  2. garf/executors/api_executor.py +228 -0
  3. garf/executors/bq_executor.py +179 -0
  4. garf/executors/config.py +52 -0
  5. garf/executors/entrypoints/__init__.py +0 -0
  6. garf/executors/entrypoints/cli.py +164 -0
  7. {garf_executors → garf/executors}/entrypoints/grpc_server.py +22 -9
  8. garf/executors/entrypoints/server.py +174 -0
  9. garf/executors/entrypoints/tracer.py +82 -0
  10. garf/executors/entrypoints/utils.py +140 -0
  11. garf/executors/exceptions.py +17 -0
  12. garf/executors/execution_context.py +117 -0
  13. garf/executors/executor.py +124 -0
  14. garf/executors/fetchers.py +128 -0
  15. garf/executors/garf_pb2.py +51 -0
  16. {garf_executors → garf/executors}/garf_pb2_grpc.py +45 -2
  17. garf/executors/query_processor.py +79 -0
  18. garf/executors/setup.py +58 -0
  19. garf/executors/sql_executor.py +144 -0
  20. garf/executors/telemetry.py +20 -0
  21. garf/executors/workflows/__init__.py +0 -0
  22. garf/executors/workflows/gcp_workflow.yaml +49 -0
  23. garf/executors/workflows/workflow.py +164 -0
  24. garf/executors/workflows/workflow_runner.py +172 -0
  25. garf_executors/__init__.py +9 -44
  26. garf_executors/api_executor.py +9 -121
  27. garf_executors/bq_executor.py +9 -161
  28. garf_executors/config.py +9 -37
  29. garf_executors/entrypoints/__init__.py +25 -0
  30. garf_executors/entrypoints/cli.py +9 -148
  31. garf_executors/entrypoints/grcp_server.py +25 -0
  32. garf_executors/entrypoints/server.py +9 -102
  33. garf_executors/entrypoints/tracer.py +8 -40
  34. garf_executors/entrypoints/utils.py +9 -124
  35. garf_executors/exceptions.py +11 -3
  36. garf_executors/execution_context.py +9 -100
  37. garf_executors/executor.py +9 -108
  38. garf_executors/fetchers.py +9 -63
  39. garf_executors/sql_executor.py +9 -125
  40. garf_executors/telemetry.py +10 -5
  41. garf_executors/workflow.py +8 -79
  42. {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/METADATA +18 -5
  43. garf_executors-1.1.3.dist-info/RECORD +46 -0
  44. {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/WHEEL +1 -1
  45. garf_executors-1.1.3.dist-info/entry_points.txt +2 -0
  46. {garf_executors-0.2.3.dist-info → garf_executors-1.1.3.dist-info}/top_level.txt +1 -0
  47. garf_executors/garf_pb2.py +0 -45
  48. garf_executors-0.2.3.dist-info/RECORD +0 -24
  49. garf_executors-0.2.3.dist-info/entry_points.txt +0 -2
@@ -0,0 +1,124 @@
1
+ # Copyright 2025 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Defines common functionality between executors."""
16
+
17
+ import asyncio
18
+ import inspect
19
+ from typing import Optional
20
+
21
+ from garf.core import report_fetcher
22
+ from garf.executors import execution_context, query_processor
23
+ from garf.executors.telemetry import tracer
24
+ from opentelemetry import trace
25
+
26
+
27
+ class Executor:
28
+ """Defines common functionality between executors."""
29
+
30
+ def __init__(
31
+ self,
32
+ preprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
33
+ postprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
34
+ ) -> None:
35
+ self.preprocessors = preprocessors or {}
36
+ self.postprocessors = postprocessors or {}
37
+
38
+ @tracer.start_as_current_span('api.execute_batch')
39
+ def execute_batch(
40
+ self,
41
+ batch: dict[str, str],
42
+ context: execution_context.ExecutionContext,
43
+ parallel_threshold: int = 10,
44
+ ) -> list[str]:
45
+ """Executes batch of queries for a common context.
46
+
47
+ If an executor has any pre/post processors, executes them first while
48
+ modifying the context.
49
+
50
+ Args:
51
+ batch: Mapping between query_title and its text.
52
+ context: Execution context.
53
+ parallel_threshold: Number of queries to execute in parallel.
54
+
55
+ Returns:
56
+ Results of execution.
57
+ """
58
+ span = trace.get_current_span()
59
+ span.set_attribute('api.parallel_threshold', parallel_threshold)
60
+ _handle_processors(processors=self.preprocessors, context=context)
61
+ results = asyncio.run(
62
+ self._run(
63
+ batch=batch, context=context, parallel_threshold=parallel_threshold
64
+ )
65
+ )
66
+ _handle_processors(processors=self.postprocessors, context=context)
67
+ return results
68
+
69
+ def add_preprocessor(
70
+ self, preprocessors: dict[str, report_fetcher.Processor]
71
+ ) -> None:
72
+ self.preprocessors.update(preprocessors)
73
+
74
+ async def aexecute(
75
+ self,
76
+ query: str,
77
+ title: str,
78
+ context: execution_context.ExecutionContext,
79
+ ) -> str:
80
+ """Performs query execution asynchronously.
81
+
82
+ Args:
83
+ query: Location of the query.
84
+ title: Name of the query.
85
+ context: Query execution context.
86
+
87
+ Returns:
88
+ Result of writing the report.
89
+ """
90
+ return await asyncio.to_thread(self.execute, query, title, context)
91
+
92
+ async def _run(
93
+ self,
94
+ batch: dict[str, str],
95
+ context: execution_context.ExecutionContext,
96
+ parallel_threshold: int,
97
+ ):
98
+ semaphore = asyncio.Semaphore(value=parallel_threshold)
99
+
100
+ async def run_with_semaphore(fn):
101
+ async with semaphore:
102
+ return await fn
103
+
104
+ tasks = [
105
+ self.aexecute(query=query, title=title, context=context)
106
+ for title, query in batch.items()
107
+ ]
108
+ return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))
109
+
110
+
111
+ def _handle_processors(
112
+ processors: dict[str, report_fetcher.Processor],
113
+ context: execution_context.ExecutionContext,
114
+ ) -> None:
115
+ context = query_processor.process_gquery(context)
116
+ for k, processor in processors.items():
117
+ processor_signature = list(inspect.signature(processor).parameters.keys())
118
+ if k in context.fetcher_parameters:
119
+ processor_parameters = {
120
+ k: v
121
+ for k, v in context.fetcher_parameters.items()
122
+ if k in processor_signature
123
+ }
124
+ context.fetcher_parameters[k] = processor(**processor_parameters)
@@ -0,0 +1,128 @@
1
+ # Copyright 2025 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ import logging
17
+ import sys
18
+ from importlib.metadata import entry_points
19
+
20
+ from garf.core import report_fetcher, simulator
21
+ from garf.executors.telemetry import tracer
22
+
23
+ logger = logging.getLogger(name='garf.executors.fetchers')
24
+
25
+
26
+ @tracer.start_as_current_span('find_fetchers')
27
+ def find_fetchers() -> set[str]:
28
+ """Identifiers all available report fetchers."""
29
+ if entrypoints := _get_entrypoints('garf'):
30
+ return {fetcher.name for fetcher in entrypoints}
31
+ return set()
32
+
33
+
34
+ @tracer.start_as_current_span('find_simulators')
35
+ def find_simulators() -> set[str]:
36
+ """Identifiers all available report simulators."""
37
+ if entrypoints := _get_entrypoints('garf_simulator'):
38
+ return {simulator.name for simulator in entrypoints}
39
+ return set()
40
+
41
+
42
+ @tracer.start_as_current_span('get_report_fetcher')
43
+ def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
44
+ """Loads report fetcher for a given source.
45
+
46
+ Args:
47
+ source: Alias for a source associated with a fetcher.
48
+
49
+ Returns:
50
+ Class for a found report fetcher.
51
+
52
+ Raises:
53
+ ApiReportFetcherError: When fetcher cannot be loaded.
54
+ MissingApiReportFetcherError: When fetcher not found.
55
+ """
56
+ if source not in find_fetchers():
57
+ raise report_fetcher.MissingApiReportFetcherError(source)
58
+ for fetcher in _get_entrypoints('garf'):
59
+ if fetcher.name == source:
60
+ try:
61
+ with tracer.start_as_current_span('load_fetcher_module') as span:
62
+ fetcher_module = fetcher.load()
63
+ span.set_attribute('loaded_module', fetcher_module.__name__)
64
+ for name, obj in inspect.getmembers(fetcher_module):
65
+ if inspect.isclass(obj) and issubclass(
66
+ obj, report_fetcher.ApiReportFetcher
67
+ ):
68
+ if not hasattr(obj, 'alias'):
69
+ return getattr(fetcher_module, name)
70
+ if obj.alias == fetcher.name:
71
+ return getattr(fetcher_module, name)
72
+ except ModuleNotFoundError as e:
73
+ raise report_fetcher.ApiReportFetcherError(
74
+ f'Failed to load fetcher for source {source}, reason: {e}'
75
+ )
76
+ raise report_fetcher.ApiReportFetcherError(
77
+ f'No fetcher available for the source "{source}"'
78
+ )
79
+
80
+
81
+ @tracer.start_as_current_span('get_report_simulator')
82
+ def get_report_simulator(source: str) -> type[simulator.ApiReportSimulator]:
83
+ """Loads report simulator for a given source.
84
+
85
+ Args:
86
+ source: Alias for a source associated with a simulator.
87
+
88
+ Returns:
89
+ Class for a found report simulator.
90
+
91
+ Raises:
92
+ GarfApiReportSimulatorError: When simulator cannot be loaded.
93
+ MissingApiReportSimulatorError: When simulator not found.
94
+ """
95
+ if source not in find_simulators():
96
+ raise simulator.MissingApiReportSimulatorError(source)
97
+ for sim in _get_entrypoints('garf_simulator'):
98
+ if sim.name == source:
99
+ try:
100
+ with tracer.start_as_current_span('load_simulator_module') as span:
101
+ simulator_module = sim.load()
102
+ span.set_attribute('loaded_module', simulator_module.__name__)
103
+ for name, obj in inspect.getmembers(simulator_module):
104
+ if inspect.isclass(obj) and issubclass(
105
+ obj, simulator.ApiReportSimulator
106
+ ):
107
+ if not hasattr(obj, 'alias'):
108
+ return getattr(simulator_module, name)
109
+ if obj.alias == sim.name:
110
+ return getattr(simulator_module, name)
111
+ except ModuleNotFoundError as e:
112
+ raise simulator.GarfApiReportSimulatorError(
113
+ f'Failed to load simulator for source {source}, reason: {e}'
114
+ )
115
+ raise simulator.GarfApiReportSimulatorError(
116
+ f'No simulator available for the source "{source}"'
117
+ )
118
+
119
+
120
+ def _get_entrypoints(group='garf'):
121
+ if sys.version_info.major == 3 and sys.version_info.minor == 9:
122
+ try:
123
+ fetchers = entry_points()[group]
124
+ except KeyError:
125
+ fetchers = []
126
+ else:
127
+ fetchers = entry_points(group=group)
128
+ return fetchers
@@ -0,0 +1,51 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
3
+ # NO CHECKED-IN PROTOBUF GENCODE
4
+ # source: garf.proto
5
+ # Protobuf Python Version: 6.31.1
6
+ """Generated protocol buffer code."""
7
+ from google.protobuf import descriptor as _descriptor
8
+ from google.protobuf import descriptor_pool as _descriptor_pool
9
+ from google.protobuf import runtime_version as _runtime_version
10
+ from google.protobuf import symbol_database as _symbol_database
11
+ from google.protobuf.internal import builder as _builder
12
+ _runtime_version.ValidateProtobufRuntimeVersion(
13
+ _runtime_version.Domain.PUBLIC,
14
+ 6,
15
+ 31,
16
+ 1,
17
+ '',
18
+ 'garf.proto'
19
+ )
20
+ # @@protoc_insertion_point(imports)
21
+
22
+ _sym_db = _symbol_database.Default()
23
+
24
+
25
+ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
26
+
27
+
28
+ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\ngarf.proto\x12\x04garf\x1a\x1cgoogle/protobuf/struct.proto\"a\n\x0c\x46\x65tchRequest\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12\r\n\x05query\x18\x03 \x01(\t\x12#\n\x07\x63ontext\x18\x04 \x01(\x0b\x32\x12.garf.FetchContext\"G\n\rFetchResponse\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\x12%\n\x04rows\x18\x02 \x03(\x0b\x32\x17.google.protobuf.Struct\"t\n\x0c\x46\x65tchContext\x12/\n\x10query_parameters\x18\x01 \x01(\x0b\x32\x15.garf.QueryParameters\x12\x33\n\x12\x66\x65tcher_parameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"g\n\x0e\x45xecuteRequest\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12\r\n\x05query\x18\x03 \x01(\t\x12\'\n\x07\x63ontext\x18\x04 \x01(\x0b\x32\x16.garf.ExecutionContext\"\xbc\x01\n\x10\x45xecutionContext\x12/\n\x10query_parameters\x18\x01 \x01(\x0b\x32\x15.garf.QueryParameters\x12\x33\n\x12\x66\x65tcher_parameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x0e\n\x06writer\x18\x03 \x01(\t\x12\x32\n\x11writer_parameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\"d\n\x0fQueryParameters\x12&\n\x05macro\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\x12)\n\x08template\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"\"\n\x0f\x45xecuteResponse\x12\x0f\n\x07results\x18\x01 \x03(\t2{\n\x0bGarfService\x12\x38\n\x07\x45xecute\x12\x14.garf.ExecuteRequest\x1a\x15.garf.ExecuteResponse\"\x00\x12\x32\n\x05\x46\x65tch\x12\x12.garf.FetchRequest\x1a\x13.garf.FetchResponse\"\x00\x62\x06proto3')
29
+
30
+ _globals = globals()
31
+ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
32
+ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'garf_pb2', _globals)
33
+ if not _descriptor._USE_C_DESCRIPTORS:
34
+ DESCRIPTOR._loaded_options = None
35
+ _globals['_FETCHREQUEST']._serialized_start=50
36
+ _globals['_FETCHREQUEST']._serialized_end=147
37
+ _globals['_FETCHRESPONSE']._serialized_start=149
38
+ _globals['_FETCHRESPONSE']._serialized_end=220
39
+ _globals['_FETCHCONTEXT']._serialized_start=222
40
+ _globals['_FETCHCONTEXT']._serialized_end=338
41
+ _globals['_EXECUTEREQUEST']._serialized_start=340
42
+ _globals['_EXECUTEREQUEST']._serialized_end=443
43
+ _globals['_EXECUTIONCONTEXT']._serialized_start=446
44
+ _globals['_EXECUTIONCONTEXT']._serialized_end=634
45
+ _globals['_QUERYPARAMETERS']._serialized_start=636
46
+ _globals['_QUERYPARAMETERS']._serialized_end=736
47
+ _globals['_EXECUTERESPONSE']._serialized_start=738
48
+ _globals['_EXECUTERESPONSE']._serialized_end=772
49
+ _globals['_GARFSERVICE']._serialized_start=774
50
+ _globals['_GARFSERVICE']._serialized_end=897
51
+ # @@protoc_insertion_point(module_scope)
@@ -5,7 +5,7 @@ import warnings
5
5
 
6
6
  from . import garf_pb2 as garf__pb2
7
7
 
8
- GRPC_GENERATED_VERSION = '1.75.0'
8
+ GRPC_GENERATED_VERSION = '1.76.0'
9
9
  GRPC_VERSION = grpc.__version__
10
10
  _version_not_supported = False
11
11
 
@@ -18,7 +18,7 @@ except ImportError:
18
18
  if _version_not_supported:
19
19
  raise RuntimeError(
20
20
  f'The grpc package installed is at version {GRPC_VERSION},'
21
- + f' but the generated code in garf_pb2_grpc.py depends on'
21
+ + ' but the generated code in garf_pb2_grpc.py depends on'
22
22
  + f' grpcio>={GRPC_GENERATED_VERSION}.'
23
23
  + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
24
24
  + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
@@ -39,6 +39,11 @@ class GarfServiceStub(object):
39
39
  request_serializer=garf__pb2.ExecuteRequest.SerializeToString,
40
40
  response_deserializer=garf__pb2.ExecuteResponse.FromString,
41
41
  _registered_method=True)
42
+ self.Fetch = channel.unary_unary(
43
+ '/garf.GarfService/Fetch',
44
+ request_serializer=garf__pb2.FetchRequest.SerializeToString,
45
+ response_deserializer=garf__pb2.FetchResponse.FromString,
46
+ _registered_method=True)
42
47
 
43
48
 
44
49
  class GarfServiceServicer(object):
@@ -50,6 +55,12 @@ class GarfServiceServicer(object):
50
55
  context.set_details('Method not implemented!')
51
56
  raise NotImplementedError('Method not implemented!')
52
57
 
58
+ def Fetch(self, request, context):
59
+ """Missing associated documentation comment in .proto file."""
60
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
61
+ context.set_details('Method not implemented!')
62
+ raise NotImplementedError('Method not implemented!')
63
+
53
64
 
54
65
  def add_GarfServiceServicer_to_server(servicer, server):
55
66
  rpc_method_handlers = {
@@ -58,6 +69,11 @@ def add_GarfServiceServicer_to_server(servicer, server):
58
69
  request_deserializer=garf__pb2.ExecuteRequest.FromString,
59
70
  response_serializer=garf__pb2.ExecuteResponse.SerializeToString,
60
71
  ),
72
+ 'Fetch': grpc.unary_unary_rpc_method_handler(
73
+ servicer.Fetch,
74
+ request_deserializer=garf__pb2.FetchRequest.FromString,
75
+ response_serializer=garf__pb2.FetchResponse.SerializeToString,
76
+ ),
61
77
  }
62
78
  generic_handler = grpc.method_handlers_generic_handler(
63
79
  'garf.GarfService', rpc_method_handlers)
@@ -95,3 +111,30 @@ class GarfService(object):
95
111
  timeout,
96
112
  metadata,
97
113
  _registered_method=True)
114
+
115
+ @staticmethod
116
+ def Fetch(request,
117
+ target,
118
+ options=(),
119
+ channel_credentials=None,
120
+ call_credentials=None,
121
+ insecure=False,
122
+ compression=None,
123
+ wait_for_ready=None,
124
+ timeout=None,
125
+ metadata=None):
126
+ return grpc.experimental.unary_unary(
127
+ request,
128
+ target,
129
+ '/garf.GarfService/Fetch',
130
+ garf__pb2.FetchRequest.SerializeToString,
131
+ garf__pb2.FetchResponse.FromString,
132
+ options,
133
+ channel_credentials,
134
+ insecure,
135
+ call_credentials,
136
+ compression,
137
+ wait_for_ready,
138
+ timeout,
139
+ metadata,
140
+ _registered_method=True)
@@ -0,0 +1,79 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """qQuery can be used as a parameter in garf queries."""
16
+
17
+ import contextlib
18
+
19
+ from garf.core import query_editor, query_parser
20
+ from garf.executors import execution_context
21
+
22
+
23
+ class GqueryError(query_parser.GarfQueryError):
24
+ """Errors on incorrect qQuery syntax."""
25
+
26
+
27
+ def _handle_sub_context(context, sub_context):
28
+ for k, v in sub_context.items():
29
+ if isinstance(v, str) and v.startswith('gquery'):
30
+ no_writer_context = context.model_copy(update={'writer': None})
31
+ try:
32
+ _, alias, *query = v.split(':', maxsplit=3)
33
+ except ValueError:
34
+ raise GqueryError(
35
+ f'Incorrect gquery format, should be gquery:alias:query, got {v}'
36
+ )
37
+ if not alias:
38
+ raise GqueryError(f'Missing alias in gquery: {v}')
39
+ if not query:
40
+ raise GqueryError(f'Missing query text in gquery: {v}')
41
+ if alias == 'sqldb':
42
+ from garf.executors import sql_executor
43
+
44
+ gquery_executor = (
45
+ sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
46
+ context.fetcher_parameters.get('connection_string')
47
+ )
48
+ )
49
+ elif alias == 'bq':
50
+ from garf.executors import bq_executor
51
+
52
+ gquery_executor = bq_executor.BigQueryExecutor(
53
+ **context.fetcher_parameters
54
+ )
55
+ else:
56
+ raise GqueryError(f'Unsupported alias {alias} for gquery: {v}')
57
+ with contextlib.suppress(
58
+ query_editor.GarfResourceError, query_parser.GarfVirtualColumnError
59
+ ):
60
+ query = ':'.join(query)
61
+ query_spec = query_editor.QuerySpecification(
62
+ text=query, args=context.query_parameters
63
+ ).generate()
64
+ if len(columns := [c for c in query_spec.column_names if c != '_']) > 1:
65
+ raise GqueryError(f'Multiple columns in gquery definition: {columns}')
66
+ res = gquery_executor.execute(
67
+ query=query, title='gquery', context=no_writer_context
68
+ )
69
+ if len(columns := [c for c in res.column_names if c != '_']) > 1:
70
+ raise GqueryError(f'Multiple columns in gquery result: {columns}')
71
+ sub_context[k] = res.to_list(row_type='scalar')
72
+
73
+
74
+ def process_gquery(
75
+ context: execution_context.ExecutionContext,
76
+ ) -> execution_context.ExecutionContext:
77
+ _handle_sub_context(context, context.fetcher_parameters)
78
+ _handle_sub_context(context, context.query_parameters.macro)
79
+ return context
@@ -0,0 +1,58 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Bootstraps executor based on provided parameters."""
15
+
16
+ from __future__ import annotations
17
+
18
+ import importlib
19
+
20
+ from garf.executors import executor, fetchers
21
+ from garf.executors.api_executor import ApiQueryExecutor
22
+ from garf.executors.telemetry import tracer
23
+
24
+
25
+ @tracer.start_as_current_span('setup_executor')
26
+ def setup_executor(
27
+ source: str,
28
+ fetcher_parameters: dict[str, str | int | bool],
29
+ enable_cache: bool = False,
30
+ cache_ttl_seconds: int = 3600,
31
+ simulate: bool = False,
32
+ ) -> type[executor.Executor]:
33
+ """Initializes executors based on a source and parameters."""
34
+ if source == 'bq':
35
+ bq_executor = importlib.import_module('garf.executors.bq_executor')
36
+ query_executor = bq_executor.BigQueryExecutor(**fetcher_parameters)
37
+ elif source == 'sqldb':
38
+ sql_executor = importlib.import_module('garf.executors.sql_executor')
39
+ query_executor = (
40
+ sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
41
+ fetcher_parameters.get('connection_string')
42
+ )
43
+ )
44
+ else:
45
+ concrete_api_fetcher = fetchers.get_report_fetcher(source)
46
+ if simulate:
47
+ concrete_simulator = fetchers.get_report_simulator(source)()
48
+ else:
49
+ concrete_simulator = None
50
+ query_executor = ApiQueryExecutor(
51
+ fetcher=concrete_api_fetcher(
52
+ **fetcher_parameters,
53
+ enable_cache=enable_cache,
54
+ cache_ttl_seconds=cache_ttl_seconds,
55
+ ),
56
+ report_simulator=concrete_simulator,
57
+ )
58
+ return query_executor
@@ -0,0 +1,144 @@
1
+ # Copyright 2024 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Defines mechanism for executing queries via SqlAlchemy."""
15
+
16
+ from __future__ import annotations
17
+
18
+ try:
19
+ import sqlalchemy
20
+ except ImportError as e:
21
+ raise ImportError(
22
+ 'Please install garf-executors with sqlalchemy support '
23
+ '- `pip install garf-executors[sqlalchemy]`'
24
+ ) from e
25
+
26
+ import logging
27
+ import re
28
+ import uuid
29
+
30
+ import pandas as pd
31
+ from garf.core import query_editor, report
32
+ from garf.executors import exceptions, execution_context, executor
33
+ from garf.executors.telemetry import tracer
34
+ from opentelemetry import trace
35
+
36
+ logger = logging.getLogger(__name__)
37
+
38
+
39
+ class SqlAlchemyQueryExecutorError(exceptions.GarfExecutorError):
40
+ """Error when SqlAlchemyQueryExecutor fails to run query."""
41
+
42
+
43
+ class SqlAlchemyQueryExecutor(
44
+ executor.Executor, query_editor.TemplateProcessorMixin
45
+ ):
46
+ """Handles query execution via SqlAlchemy.
47
+
48
+ Attributes:
49
+ engine: Initialized Engine object to operated on a given database.
50
+ """
51
+
52
+ def __init__(
53
+ self, engine: sqlalchemy.engine.base.Engine | None = None, **kwargs: str
54
+ ) -> None:
55
+ """Initializes executor with a given engine.
56
+
57
+ Args:
58
+ engine: Initialized Engine object to operated on a given database.
59
+ """
60
+ self.engine = engine or sqlalchemy.create_engine('sqlite://')
61
+ super().__init__()
62
+
63
+ @classmethod
64
+ def from_connection_string(
65
+ cls, connection_string: str | None
66
+ ) -> SqlAlchemyQueryExecutor:
67
+ """Creates executor from SqlAlchemy connection string.
68
+
69
+ https://docs.sqlalchemy.org/en/20/core/engines.html
70
+ """
71
+ engine = sqlalchemy.create_engine(connection_string or 'sqlite://')
72
+ return cls(engine)
73
+
74
+ @tracer.start_as_current_span('sql.execute')
75
+ def execute(
76
+ self,
77
+ query: str,
78
+ title: str,
79
+ context: execution_context.ExecutionContext = (
80
+ execution_context.ExecutionContext()
81
+ ),
82
+ ) -> report.GarfReport:
83
+ """Executes query in a given database via SqlAlchemy.
84
+
85
+ Args:
86
+ query: Location of the query.
87
+ title: Name of the query.
88
+ context: Query execution context.
89
+
90
+ Returns:
91
+ Report with data if query returns some data otherwise empty Report.
92
+ """
93
+ span = trace.get_current_span()
94
+ span.set_attribute('query.title', title)
95
+ span.set_attribute('query.text', query)
96
+ logger.info('Executing script: %s', title)
97
+ query_text = self.replace_params_template(query, context.query_parameters)
98
+ with self.engine.begin() as conn:
99
+ if re.findall(r'(create|update) ', query_text.lower()):
100
+ try:
101
+ conn.connection.executescript(query_text)
102
+ results = report.GarfReport()
103
+ except Exception as e:
104
+ raise SqlAlchemyQueryExecutorError(
105
+ f'Failed to execute query {title}: Reason: {e}'
106
+ ) from e
107
+ else:
108
+ temp_table_name = f'temp_{uuid.uuid4().hex}'
109
+ query_text = f'CREATE TABLE {temp_table_name} AS {query_text}'
110
+ conn.connection.executescript(query_text)
111
+ try:
112
+ results = report.GarfReport.from_pandas(
113
+ pd.read_sql(f'SELECT * FROM {temp_table_name}', conn)
114
+ )
115
+ except Exception as e:
116
+ raise SqlAlchemyQueryExecutorError(
117
+ f'Failed to execute query {title}: Reason: {e}'
118
+ ) from e
119
+ finally:
120
+ conn.connection.execute(f'DROP TABLE {temp_table_name}')
121
+ if context.writer and results:
122
+ writer_clients = context.writer_clients
123
+ if not writer_clients:
124
+ logger.warning('No writers configured, skipping write operation')
125
+ else:
126
+ writing_results = []
127
+ for writer_client in writer_clients:
128
+ logger.debug(
129
+ 'Start writing data for query %s via %s writer',
130
+ title,
131
+ type(writer_client),
132
+ )
133
+ writing_result = writer_client.write(results, title)
134
+ logger.debug(
135
+ 'Finish writing data for query %s via %s writer',
136
+ title,
137
+ type(writer_client),
138
+ )
139
+ writing_results.append(writing_result)
140
+ logger.info('%s executed successfully', title)
141
+ # Return the last writer's result for backward compatibility
142
+ return writing_results[-1] if writing_results else None
143
+ span.set_attribute('execute.num_results', len(results))
144
+ return results
@@ -0,0 +1,20 @@
1
+ # Copyright 2025 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # pylint: disable=C0330, g-bad-import-order, g-multiple-import
16
+ from opentelemetry import trace
17
+
18
+ tracer = trace.get_tracer(
19
+ instrumenting_module_name='garf.executors',
20
+ )