garf-executors 0.1.4__py3-none-any.whl → 0.2.3__py3-none-any.whl

garf_executors/__init__.py

@@ -43,7 +43,7 @@ def setup_executor(
   else:
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
     query_executor = ApiQueryExecutor(
-      concrete_api_fetcher(
+      fetcher=concrete_api_fetcher(
         **fetcher_parameters,
         enable_cache=enable_cache,
         cache_ttl_seconds=cache_ttl_seconds,
@@ -57,4 +57,4 @@ __all__ = [
   'ApiExecutionContext',
 ]
 
-__version__ = '0.1.4'
+__version__ = '0.2.3'

garf_executors/api_executor.py

@@ -20,7 +20,6 @@ GarfReport and saving it to local/remote storage.
 
 from __future__ import annotations
 
-import asyncio
 import logging
 
 from garf_core import report_fetcher
@@ -35,7 +34,7 @@ logger = logging.getLogger(__name__)
 class ApiExecutionContext(execution_context.ExecutionContext):
   """Common context for executing one or more queries."""
 
-  writer: str = 'console'
+  writer: str | list[str] = 'console'
 
 
 class ApiQueryExecutor(executor.Executor):
@@ -52,15 +51,29 @@ class ApiQueryExecutor(executor.Executor):
         fetcher: Instantiated report fetcher.
     """
     self.fetcher = fetcher
+    super().__init__(
+      preprocessors=self.fetcher.preprocessors,
+      postprocessors=self.fetcher.postprocessors,
+    )
 
   @classmethod
   def from_fetcher_alias(
-    cls, source: str, fetcher_parameters: dict[str, str] | None = None
+    cls,
+    source: str,
+    fetcher_parameters: dict[str, str] | None = None,
+    enable_cache: bool = False,
+    cache_ttl_seconds: int = 3600,
   ) -> ApiQueryExecutor:
     if not fetcher_parameters:
       fetcher_parameters = {}
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    return ApiQueryExecutor(concrete_api_fetcher(**fetcher_parameters))
+    return ApiQueryExecutor(
+      fetcher=concrete_api_fetcher(
+        **fetcher_parameters,
+        enable_cache=enable_cache,
+        cache_ttl_seconds=cache_ttl_seconds,
+      )
+    )
 
   @tracer.start_as_current_span('api.execute')
   def execute(
@@ -83,31 +96,40 @@ class ApiQueryExecutor(executor.Executor):
       GarfExecutorError: When failed to execute query.
     """
     span = trace.get_current_span()
-    span.set_attribute('fetcher', self.fetcher.__class__.__name__)
-    span.set_attribute('api_client', self.fetcher.api_client.__class__.__name__)
+    span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
+    span.set_attribute(
+      'api.client.class', self.fetcher.api_client.__class__.__name__
+    )
     try:
-      span.set_attribute('query_title', title)
-      span.set_attribute('query_text', query)
+      span.set_attribute('query.title', title)
+      span.set_attribute('query.text', query)
       logger.debug('starting query %s', query)
       results = self.fetcher.fetch(
         query_specification=query,
         args=context.query_parameters,
         **context.fetcher_parameters,
       )
-      writer_client = context.writer_client
-      logger.debug(
-        'Start writing data for query %s via %s writer',
-        title,
-        type(writer_client),
-      )
-      result = writer_client.write(results, title)
-      logger.debug(
-        'Finish writing data for query %s via %s writer',
-        title,
-        type(writer_client),
-      )
+      writer_clients = context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
+        return None
+      writing_results = []
+      for writer_client in writer_clients:
+        logger.debug(
+          'Start writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        result = writer_client.write(results, title)
+        logger.debug(
+          'Finish writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        writing_results.append(result)
       logger.info('%s executed successfully', title)
-      return result
+      # Return the last writer's result for backward compatibility
+      return writing_results[-1] if writing_results else None
     except Exception as e:
       logger.error('%s generated an exception: %s', title, str(e))
       raise exceptions.GarfExecutorError(

garf_executors/bq_executor.py

@@ -15,6 +15,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 
 try:
@@ -29,6 +30,7 @@ import logging
 
 from garf_core import query_editor, report
 from google.cloud import exceptions as google_cloud_exceptions
+from opentelemetry import trace
 
 from garf_executors import exceptions, execution_context, executor
 from garf_executors.telemetry import tracer
@@ -67,6 +69,7 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
       )
     self.project_id = project_id
     self.location = location
+    super().__init__()
 
   @property
   def client(self) -> bigquery.Client:
@@ -92,34 +95,47 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
     Returns:
       Report with data if query returns some data otherwise empty Report.
     """
+    span = trace.get_current_span()
+    logger.info('Executing script: %s', title)
     query_text = self.replace_params_template(query, context.query_parameters)
     self.create_datasets(context.query_parameters.macro)
     job = self.client.query(query_text)
     try:
       result = job.result()
+    except google_cloud_exceptions.GoogleCloudError as e:
+      raise BigQueryExecutorError(
+        f'Failed to execute query {title}: Reason: {e}'
+      ) from e
       logger.debug('%s launched successfully', title)
-      if result.total_rows:
-        results = report.GarfReport.from_pandas(result.to_dataframe())
+    if result.total_rows:
+      results = report.GarfReport.from_pandas(result.to_dataframe())
+    else:
+      results = report.GarfReport()
+    if context.writer and results:
+      writer_clients = context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
       else:
-        results = report.GarfReport()
-      if context.writer and results:
-        writer_client = context.writer_client
-        logger.debug(
-          'Start writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        writing_result = writer_client.write(results, title)
-        logger.debug(
-          'Finish writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
+        writing_results = []
+        for writer_client in writer_clients:
+          logger.debug(
+            'Start writing data for query %s via %s writer',
+            title,
+            type(writer_client),
+          )
+          writing_result = writer_client.write(results, title)
+          logger.debug(
+            'Finish writing data for query %s via %s writer',
+            title,
+            type(writer_client),
+          )
+          writing_results.append(writing_result)
+        # Return the last writer's result for backward compatibility
         logger.info('%s executed successfully', title)
-        return writing_result
-      return results
-    except google_cloud_exceptions.GoogleCloudError as e:
-      raise BigQueryExecutorError(e) from e
+        return writing_results[-1] if writing_results else None
+    logger.info('%s executed successfully', title)
+    span.set_attribute('execute.num_results', len(results))
+    return results
 
   @tracer.start_as_current_span('bq.create_datasets')
   def create_datasets(self, macros: dict | None) -> None:
@@ -139,8 +155,9 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
         except google_cloud_exceptions.NotFound:
           bq_dataset = bigquery.Dataset(dataset_id)
           bq_dataset.location = self.location
-          self.client.create_dataset(bq_dataset, timeout=30)
-          logger.info('Created new dataset %s', dataset_id)
+          with contextlib.suppress(google_cloud_exceptions.Conflict):
+            self.client.create_dataset(bq_dataset, timeout=30)
+            logger.info('Created new dataset %s', dataset_id)
 
 
 def extract_datasets(macros: dict | None) -> list[str]:

garf_executors/config.py

@@ -47,5 +47,7 @@ class Config(pydantic.BaseModel):
   def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
     """Saves config to local or remote yaml file."""
     with smart_open.open(path, 'w', encoding='utf-8') as f:
-      yaml.dump(self.model_dump().get('sources'), f, encoding='utf-8')
+      yaml.dump(
+        self.model_dump(exclude_none=True).get('sources'), f, encoding='utf-8'
+      )
     return f'Config is saved to {str(path)}'

garf_executors/entrypoints/cli.py

@@ -24,9 +24,10 @@ import logging
 import sys
 
 from garf_io import reader
+from opentelemetry import trace
 
 import garf_executors
-from garf_executors import config, exceptions
+from garf_executors import config, exceptions, workflow
 from garf_executors.entrypoints import utils
 from garf_executors.entrypoints.tracer import initialize_tracer
 from garf_executors.telemetry import tracer
@@ -39,6 +40,7 @@ def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('query', nargs='*')
   parser.add_argument('-c', '--config', dest='config', default=None)
+  parser.add_argument('-w', '--workflow', dest='workflow', default=None)
   parser.add_argument('--source', dest='source', default=None)
   parser.add_argument('--output', dest='output', default='console')
   parser.add_argument('--input', dest='input', default='file')
@@ -70,61 +72,91 @@ def main():
   parser.set_defaults(dry_run=False)
   args, kwargs = parser.parse_known_args()
 
+  span = trace.get_current_span()
+  command_args = ' '.join(sys.argv[1:])
+  span.set_attribute('cli.command', f'garf {command_args}')
   if args.version:
     print(garf_executors.__version__)
     sys.exit()
   logger = utils.init_logging(
     loglevel=args.loglevel.upper(), logger_type=args.logger, name=args.log_name
   )
+  reader_client = reader.create_reader(args.input)
+  if workflow_file := args.workflow:
+    execution_workflow = workflow.Workflow.from_file(workflow_file)
+    for i, step in enumerate(execution_workflow.steps, 1):
+      with tracer.start_as_current_span(f'{i}-{step.fetcher}'):
+        query_executor = garf_executors.setup_executor(
+          source=step.fetcher,
+          fetcher_parameters=step.fetcher_parameters,
+          enable_cache=args.enable_cache,
+          cache_ttl_seconds=args.cache_ttl_seconds,
+        )
+        batch = {}
+        if not (queries := step.queries):
+          logger.error('Please provide one or more queries to run')
+          raise exceptions.GarfExecutorError(
+            'Please provide one or more queries to run'
+          )
+        for query in queries:
+          if isinstance(query, garf_executors.workflow.QueryPath):
+            batch[query.path] = reader_client.read(query.path)
+          else:
+            batch[query.query.title] = query.query.text
+        query_executor.execute_batch(
+          batch, step.context, args.parallel_threshold
+        )
+    sys.exit()
+
   if not args.query:
     logger.error('Please provide one or more queries to run')
     raise exceptions.GarfExecutorError(
       'Please provide one or more queries to run'
     )
-  reader_client = reader.create_reader(args.input)
   if config_file := args.config:
     execution_config = config.Config.from_file(config_file)
     if not (context := execution_config.sources.get(args.source)):
       raise exceptions.GarfExecutorError(
         f'No execution context found for source {args.source} in {config_file}'
       )
-    query_executor = garf_executors.setup_executor(
-      source=args.source,
-      fetcher_parameters=context.fetcher_parameters,
-      enable_cache=args.enable_cache,
-      cache_ttl_seconds=args.cache_ttl_seconds,
-    )
-    batch = {query: reader_client.read(query) for query in args.query}
-    query_executor.execute_batch(batch, context, args.parallel_threshold)
   else:
-    extra_parameters = utils.ParamsParser(
-      ['source', args.output, 'macro', 'template']
-    ).parse(kwargs)
+    param_types = ['source', 'macro', 'template']
+    outputs = args.output.split(',')
+    extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(
+      kwargs
+    )
     source_parameters = extra_parameters.get('source', {})
+    writer_parameters = {}
+    for output in outputs:
+      writer_parameters.update(extra_parameters.get(output))
 
     context = garf_executors.api_executor.ApiExecutionContext(
       query_parameters={
         'macro': extra_parameters.get('macro'),
         'template': extra_parameters.get('template'),
       },
-      writer=args.output,
-      writer_parameters=extra_parameters.get(args.output),
+      writer=outputs,
+      writer_parameters=writer_parameters,
       fetcher_parameters=source_parameters,
     )
-    query_executor = garf_executors.setup_executor(
-      source=args.source,
-      fetcher_parameters=context.fetcher_parameters,
-      enable_cache=args.enable_cache,
-      cache_ttl_seconds=args.cache_ttl_seconds,
-    )
-    if args.parallel_queries:
-      logger.info('Running queries in parallel')
-      batch = {query: reader_client.read(query) for query in args.query}
-      query_executor.execute_batch(batch, context, args.parallel_threshold)
-    else:
+  query_executor = garf_executors.setup_executor(
+    source=args.source,
+    fetcher_parameters=context.fetcher_parameters,
+    enable_cache=args.enable_cache,
+    cache_ttl_seconds=args.cache_ttl_seconds,
+  )
+  batch = {query: reader_client.read(query) for query in args.query}
+  if args.parallel_queries and len(args.query) > 1:
+    logger.info('Running queries in parallel')
+    batch = {query: reader_client.read(query) for query in args.query}
+    query_executor.execute_batch(batch, context, args.parallel_threshold)
+  else:
+    if len(args.query) > 1:
       logger.info('Running queries sequentially')
-      for query in args.query:
-        query_executor.execute(reader_client.read(query), query, context)
+    for query in args.query:
+      query_executor.execute(
+        query=reader_client.read(query), title=query, context=context
+      )
   logging.shutdown()
 
 

garf_executors/entrypoints/grpc_server.py

@@ -0,0 +1,68 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""gRPC endpoint for garf."""
+
+import argparse
+import logging
+from concurrent import futures
+
+import grpc
+from google.protobuf.json_format import MessageToDict
+from grpc_reflection.v1alpha import reflection
+
+import garf_executors
+from garf_executors import garf_pb2, garf_pb2_grpc
+from garf_executors.entrypoints.tracer import initialize_tracer
+
+
+class GarfService(garf_pb2_grpc.GarfService):
+  def Execute(self, request, context):
+    query_executor = garf_executors.setup_executor(
+      request.source, request.context.fetcher_parameters
+    )
+    execution_context = garf_executors.execution_context.ExecutionContext(
+      **MessageToDict(request.context, preserving_proto_field_name=True)
+    )
+    result = query_executor.execute(
+      query=request.query,
+      title=request.title,
+      context=execution_context,
+    )
+    return garf_pb2.ExecuteResponse(results=[result])
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--port', dest='port', default=50051, type=int)
+  parser.add_argument(
+    '--parallel-threshold', dest='parallel_threshold', default=10, type=int
+  )
+  args, _ = parser.parse_known_args()
+  initialize_tracer()
+  server = grpc.server(
+    futures.ThreadPoolExecutor(max_workers=args.parallel_threshold)
+  )
+
+  service = GarfService()
+  garf_pb2_grpc.add_GarfServiceServicer_to_server(service, server)
+  SERVICE_NAMES = (
+    garf_pb2.DESCRIPTOR.services_by_name['GarfService'].full_name,
+    reflection.SERVICE_NAME,
+  )
+  reflection.enable_server_reflection(SERVICE_NAMES, server)
+  server.add_insecure_port(f'[::]:{args.port}')
+  server.start()
+  logging.info('Garf service started, listening on port %d', 50051)
+  server.wait_for_termination()

garf_executors/entrypoints/server.py

@@ -18,9 +18,11 @@ from typing import Optional, Union
 
 import fastapi
 import pydantic
+import typer
 import uvicorn
 from garf_io import reader
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from typing_extensions import Annotated
 
 import garf_executors
 from garf_executors import exceptions
@@ -29,6 +31,7 @@ from garf_executors.entrypoints.tracer import initialize_tracer
 initialize_tracer()
 app = fastapi.FastAPI()
 FastAPIInstrumentor.instrument_app(app)
+typer_app = typer.Typer()
 
 
 class ApiExecutorRequest(pydantic.BaseModel):
@@ -104,5 +107,12 @@ def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   return ApiExecutorResponse(results=results)
 
 
+@typer_app.command()
+def main(
+  port: Annotated[int, typer.Option(help='Port to start the server')] = 8000,
+):
+  uvicorn.run(app, port=port)
+
+
 if __name__ == '__main__':
-  uvicorn.run(app)
+  typer_app()

garf_executors/entrypoints/tracer.py

@@ -1,4 +1,4 @@
-# Copyright 2025 Google LLC
+# Copyright 2026 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,8 +35,23 @@ def initialize_tracer():
   tracer_provider = TracerProvider(resource=resource)
 
   if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
-    otlp_processor = BatchSpanProcessor(
-      OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
-    )
-    tracer_provider.add_span_processor(otlp_processor)
+    if gcp_project_id := os.getenv('OTEL_EXPORTER_GCP_PROJECT_ID'):
+      try:
+        from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+      except ImportError as e:
+        raise ImportError(
+          'Please install garf_executors with GCP support '
+          '- `pip install garf_executors[gcp]`'
+        ) from e
+
+      cloud_span_processor = BatchSpanProcessor(
+        CloudTraceSpanExporter(project_id=gcp_project_id)
+      )
+      tracer_provider.add_span_processor(cloud_span_processor)
+    else:
+      otlp_processor = BatchSpanProcessor(
+        OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
+      )
+      tracer_provider.add_span_processor(otlp_processor)
+
   trace.set_tracer_provider(tracer_provider)

garf_executors/execution_context.py

@@ -35,7 +35,7 @@ class ExecutionContext(pydantic.BaseModel):
   Attributes:
     query_parameters: Parameters to dynamically change query text.
     fetcher_parameters: Parameters to specify fetching setup.
-    writer: Type of writer to use.
+    writer: Type of writer to use. Can be a single writer string or list of writers.
     writer_parameters: Optional parameters to setup writer.
   """
 
@@ -45,7 +45,7 @@ class ExecutionContext(pydantic.BaseModel):
   fetcher_parameters: dict[str, str | bool | int | list[str | int]] | None = (
     pydantic.Field(default_factory=dict)
   )
-  writer: str | None = None
+  writer: str | list[str] | None = None
   writer_parameters: dict[str, str] | None = pydantic.Field(
     default_factory=dict
   )
@@ -75,9 +75,42 @@ class ExecutionContext(pydantic.BaseModel):
 
   @property
   def writer_client(self) -> abs_writer.AbsWriter:
-    writer_client = writer.create_writer(self.writer, **self.writer_parameters)
-    if self.writer == 'bq':
+    """Returns single writer client."""
+    if isinstance(self.writer, list) and len(self.writer) > 0:
+      writer_type = self.writer[0]
+    else:
+      writer_type = self.writer
+
+    writer_params = self.writer_parameters or {}
+
+    if not writer_type:
+      raise ValueError('No writer specified')
+
+    writer_client = writer.create_writer(writer_type, **writer_params)
+    if writer_type == 'bq':
       _ = writer_client.create_or_get_dataset()
-    if self.writer == 'sheet':
+    if writer_type == 'sheet':
       writer_client.init_client()
     return writer_client
+
+  @property
+  def writer_clients(self) -> list[abs_writer.AbsWriter]:
+    """Returns list of writer clients."""
+    if not self.writer:
+      return []
+
+    # Convert single writer to list for uniform processing
+    writers_to_use = (
+      self.writer if isinstance(self.writer, list) else [self.writer]
+    )
+    writer_params = self.writer_parameters or {}
+
+    clients = []
+    for writer_type in writers_to_use:
+      writer_client = writer.create_writer(writer_type, **writer_params)
+      if writer_type == 'bq':
+        _ = writer_client.create_or_get_dataset()
+      if writer_type == 'sheet':
+        writer_client.init_client()
+      clients.append(writer_client)
+    return clients

garf_executors/executor.py

@@ -15,7 +15,10 @@
 """Defines common functionality between executors."""
 
 import asyncio
+import inspect
+from typing import Optional
 
+from garf_core import report_fetcher
 from opentelemetry import trace
 
 from garf_executors import execution_context
@@ -25,6 +28,14 @@ from garf_executors.telemetry import tracer
 class Executor:
   """Defines common functionality between executors."""
 
+  def __init__(
+    self,
+    preprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
+    postprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
+  ) -> None:
+    self.preprocessors = preprocessors or {}
+    self.postprocessors = postprocessors or {}
+
   @tracer.start_as_current_span('api.execute_batch')
   def execute_batch(
     self,
@@ -34,6 +45,9 @@ class Executor:
   ) -> list[str]:
     """Executes batch of queries for a common context.
 
+    If an executor has any pre/post processors, executes them first while
+    modifying the context.
+
     Args:
       batch: Mapping between query_title and its text.
       context: Execution context.
@@ -44,11 +58,19 @@ class Executor:
     """
     span = trace.get_current_span()
     span.set_attribute('api.parallel_threshold', parallel_threshold)
-    return asyncio.run(
+    _handle_processors(processors=self.preprocessors, context=context)
+    results = asyncio.run(
       self._run(
         batch=batch, context=context, parallel_threshold=parallel_threshold
       )
     )
+    _handle_processors(processors=self.postprocessors, context=context)
+    return results
+
+  def add_preprocessor(
+    self, preprocessors: dict[str, report_fetcher.Processor]
+  ) -> None:
+    self.preprocessors.update(preprocessors)
 
   async def aexecute(
     self,
@@ -85,3 +107,18 @@ class Executor:
       for title, query in batch.items()
     ]
     return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))
+
+
+def _handle_processors(
+  processors: dict[str, report_fetcher.Processor],
+  context: execution_context.ExecutionContext,
+) -> None:
+  for k, processor in processors.items():
+    processor_signature = list(inspect.signature(processor).parameters.keys())
+    if k in context.fetcher_parameters:
+      processor_parameters = {
+        k: v
+        for k, v in context.fetcher_parameters.items()
+        if k in processor_signature
+      }
+      context.fetcher_parameters[k] = processor(**processor_parameters)

garf_executors/fetchers.py

@@ -13,14 +13,16 @@
 # limitations under the License.
 
 import inspect
+import logging
 import sys
 from importlib.metadata import entry_points
 
 from garf_core import report_fetcher
-from opentelemetry import trace
 
 from garf_executors.telemetry import tracer
 
+logger = logging.getLogger(name='garf_executors.fetchers')
+
 
 @tracer.start_as_current_span('find_fetchers')
 def find_fetchers() -> set[str]:
@@ -57,8 +59,10 @@ def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
             obj, report_fetcher.ApiReportFetcher
           ):
             return getattr(fetcher_module, name)
-      except ModuleNotFoundError:
-        continue
+      except ModuleNotFoundError as e:
+        raise report_fetcher.ApiReportFetcherError(
+          f'Failed to load fetcher for source {source}, reason: {e}'
+        )
   raise report_fetcher.ApiReportFetcherError(
     f'No fetcher available for the source "{source}"'
   )

garf_executors/garf_pb2.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: garf.proto
+# Protobuf Python Version: 6.31.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    31,
+    1,
+    '',
+    'garf.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\ngarf.proto\x12\x04garf\x1a\x1cgoogle/protobuf/struct.proto\"g\n\x0e\x45xecuteRequest\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12\r\n\x05query\x18\x03 \x01(\t\x12\'\n\x07\x63ontext\x18\x04 \x01(\x0b\x32\x16.garf.ExecutionContext\"\xbc\x01\n\x10\x45xecutionContext\x12/\n\x10query_parameters\x18\x01 \x01(\x0b\x32\x15.garf.QueryParameters\x12\x33\n\x12\x66\x65tcher_parameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x0e\n\x06writer\x18\x03 \x01(\t\x12\x32\n\x11writer_parameters\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\"d\n\x0fQueryParameters\x12&\n\x05macro\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\x12)\n\x08template\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"\"\n\x0f\x45xecuteResponse\x12\x0f\n\x07results\x18\x01 \x03(\t2G\n\x0bGarfService\x12\x38\n\x07\x45xecute\x12\x14.garf.ExecuteRequest\x1a\x15.garf.ExecuteResponse\"\x00\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'garf_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_EXECUTEREQUEST']._serialized_start=50
+  _globals['_EXECUTEREQUEST']._serialized_end=153
+  _globals['_EXECUTIONCONTEXT']._serialized_start=156
+  _globals['_EXECUTIONCONTEXT']._serialized_end=344
+  _globals['_QUERYPARAMETERS']._serialized_start=346
+  _globals['_QUERYPARAMETERS']._serialized_end=446
+  _globals['_EXECUTERESPONSE']._serialized_start=448
+  _globals['_EXECUTERESPONSE']._serialized_end=482
+  _globals['_GARFSERVICE']._serialized_start=484
+  _globals['_GARFSERVICE']._serialized_end=555
+# @@protoc_insertion_point(module_scope)

garf_executors/garf_pb2_grpc.py

@@ -0,0 +1,97 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+import warnings
+
+from . import garf_pb2 as garf__pb2
+
+GRPC_GENERATED_VERSION = '1.75.0'
+GRPC_VERSION = grpc.__version__
+_version_not_supported = False
+
+try:
+    from grpc._utilities import first_version_is_lower
+    _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
+except ImportError:
+    _version_not_supported = True
+
+if _version_not_supported:
+    raise RuntimeError(
+        f'The grpc package installed is at version {GRPC_VERSION},'
+        + f' but the generated code in garf_pb2_grpc.py depends on'
+        + f' grpcio>={GRPC_GENERATED_VERSION}.'
+        + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
+        + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
+    )
+
+
+class GarfServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Execute = channel.unary_unary(
+                '/garf.GarfService/Execute',
+                request_serializer=garf__pb2.ExecuteRequest.SerializeToString,
+                response_deserializer=garf__pb2.ExecuteResponse.FromString,
+                _registered_method=True)
+
+
+class GarfServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Execute(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_GarfServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Execute': grpc.unary_unary_rpc_method_handler(
+                    servicer.Execute,
+                    request_deserializer=garf__pb2.ExecuteRequest.FromString,
+                    response_serializer=garf__pb2.ExecuteResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'garf.GarfService', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('garf.GarfService', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class GarfService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Execute(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/garf.GarfService/Execute',
+            garf__pb2.ExecuteRequest.SerializeToString,
+            garf__pb2.ExecuteResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)

garf_executors/sql_executor.py

@@ -25,6 +25,7 @@ except ImportError as e:
 
 import logging
 import re
+import uuid
 
 import pandas as pd
 from garf_core import query_editor, report
@@ -56,6 +57,7 @@ class SqlAlchemyQueryExecutor(
         engine: Initialized Engine object to operated on a given database.
     """
     self.engine = engine
+    super().__init__()
 
   @classmethod
   def from_connection_string(
@@ -88,36 +90,52 @@ class SqlAlchemyQueryExecutor(
       Report with data if query returns some data otherwise empty Report.
     """
     span = trace.get_current_span()
-    logging.info('Executing script: %s', title)
+    logger.info('Executing script: %s', title)
     query_text = self.replace_params_template(query, context.query_parameters)
     with self.engine.begin() as conn:
       if re.findall(r'(create|update) ', query_text.lower()):
-        conn.connection.executescript(query_text)
-        results = report.GarfReport()
+        try:
+          conn.connection.executescript(query_text)
+          results = report.GarfReport()
+        except Exception as e:
+          raise SqlAlchemyQueryExecutorError(
+            f'Failed to execute query {title}: Reason: {e}'
+          ) from e
       else:
-        temp_table_name = f'temp_{title}'.replace('.', '_')
+        temp_table_name = f'temp_{uuid.uuid4().hex}'
         query_text = f'CREATE TABLE {temp_table_name} AS {query_text}'
         conn.connection.executescript(query_text)
         try:
           results = report.GarfReport.from_pandas(
             pd.read_sql(f'SELECT * FROM {temp_table_name}', conn)
           )
+        except Exception as e:
+          raise SqlAlchemyQueryExecutorError(
+            f'Failed to execute query {title}: Reason: {e}'
+          ) from e
         finally:
           conn.connection.execute(f'DROP TABLE {temp_table_name}')
       if context.writer and results:
-        writer_client = context.writer_client
-        logger.debug(
-          'Start writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        writing_result = writer_client.write(results, title)
-        logger.debug(
-          'Finish writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        logger.info('%s executed successfully', title)
-        return writing_result
+        writer_clients = context.writer_clients
+        if not writer_clients:
+          logger.warning('No writers configured, skipping write operation')
+        else:
+          writing_results = []
+          for writer_client in writer_clients:
+            logger.debug(
+              'Start writing data for query %s via %s writer',
+              title,
+              type(writer_client),
+            )
+            writing_result = writer_client.write(results, title)
+            logger.debug(
+              'Finish writing data for query %s via %s writer',
+              title,
+              type(writer_client),
+            )
+            writing_results.append(writing_result)
+          logger.info('%s executed successfully', title)
+          # Return the last writer's result for backward compatibility
+          return writing_results[-1] if writing_results else None
       span.set_attribute('execute.num_results', len(results))
       return results

garf_executors/workflow.py

@@ -0,0 +1,96 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pydantic
+import smart_open
+import yaml
+
+from garf_executors.execution_context import ExecutionContext
+
+
+class QueryPath(pydantic.BaseModel):
+  """Path file with query."""
+
+  path: str
+
+
+class QueryDefinition(pydantic.BaseModel):
+  """Definition of a query."""
+
+  query: Query
+
+
+class Query(pydantic.BaseModel):
+  """Query elements.
+
+  Attributes:
+    text: Query text.
+    title: Name of the query.
+  """
+
+  text: str
+  title: str
+
+
+class ExecutionStep(ExecutionContext):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    fetcher: Name of a fetcher to get data from API.
+    alias: Optional alias to identify execution step.
+    queries: Queries to run for a particular fetcher.
+    context: Execution context for queries and fetcher.
+  """
+
+  fetcher: str | None = None
+  alias: str | None = None
+  queries: list[QueryPath | QueryDefinition] | None = None
+
+  @property
+  def context(self) -> ExecutionContext:
+    return ExecutionContext(
+      writer=self.writer,
+      writer_parameters=self.writer_parameters,
+      query_parameters=self.query_parameters,
+      fetcher_parameters=self.fetcher_parameters,
+    )
+
+
+class Workflow(pydantic.BaseModel):
+  """Orchestrates execution of queries for multiple fetchers.
+
+  Attributes:
+    steps: Contains one or several fetcher executions.
+  """
+
+  steps: list[ExecutionStep]
+
+  @classmethod
+  def from_file(cls, path: str | pathlib.Path | os.PathLike[str]) -> Workflow:
+    """Builds workflow from local or remote yaml file."""
+    with smart_open.open(path, 'r', encoding='utf-8') as f:
+      data = yaml.safe_load(f)
+    return Workflow(steps=data.get('steps'))
+
+  def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
+    """Saves workflow to local or remote yaml file."""
+    with smart_open.open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(
+        self.model_dump(exclude_none=True).get('steps'), f, encoding='utf-8'
+      )
+    return f'Workflow is saved to {str(path)}'

{garf_executors-0.1.4.dist-info → garf_executors-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.1.4
+Version: 0.2.3
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0
@@ -23,19 +23,23 @@ Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: opentelemetry-api
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: opentelemetry-exporter-otlp
 Provides-Extra: bq
 Requires-Dist: garf-io[bq]; extra == "bq"
 Requires-Dist: pandas; extra == "bq"
 Requires-Dist: google-cloud-logging; extra == "bq"
+Requires-Dist: smart_open[gcs]; extra == "bq"
 Provides-Extra: sql
 Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
 Requires-Dist: pandas; extra == "sql"
+Provides-Extra: gcp
+Requires-Dist: opentelemetry-exporter-gcp-trace; extra == "gcp"
 Provides-Extra: server
 Requires-Dist: fastapi[standard]; extra == "server"
 Requires-Dist: opentelemetry-instrumentation-fastapi; extra == "server"
-Requires-Dist: opentelemetry-exporter-otlp; extra == "server"
+Requires-Dist: typer; extra == "server"
 Provides-Extra: all
-Requires-Dist: garf-executors[bq,server,sql]; extra == "all"
+Requires-Dist: garf-executors[bq,gcp,server,sql]; extra == "all"
 
 # `garf-executors` - One stop-shop for interacting with Reporting APIs.
 

garf_executors-0.2.3.dist-info/RECORD

@@ -0,0 +1,24 @@
+garf_executors/__init__.py,sha256=wZdLw0WyAGEK1y0Fdagvdd5xOWNKaPvL95yuaYziIWE,1941
+garf_executors/api_executor.py,sha256=IKYI1TK2HI2njxw7_X9n78wAQ1briAXxbA15Ybmt6nA,4295
+garf_executors/bq_executor.py,sha256=HKFBg4PhIaKM_SvjQy-ZbP7AsrsAF1FIj_w9gRqdICA,5756
+garf_executors/config.py,sha256=rZTAuBEa-Loi3DSamXFTjFQXHdeYJv71WOEbLLeo3l4,1721
+garf_executors/exceptions.py,sha256=U_7Q2ZMOUf89gzZd2pw7y3g7i1NeByPPKfpZ3q7p3ZU,662
+garf_executors/execution_context.py,sha256=WhHoN60vyeBUJbdjtOEZC1vUEyLwnIzBHbhT8co3yhs,3850
+garf_executors/executor.py,sha256=tobjdlOaAsc-nKLFSW-3qib5-ca6aHs5iw3Gn0sD72Y,3762
+garf_executors/fetchers.py,sha256=0bYurZs5jzxfGP9BgDnifdM6yRFvyCtKO-i3hFb5T5A,2605
+garf_executors/garf_pb2.py,sha256=mYvBYcAnZtyDflXGN2GZLM2KM0Nv9hoJs55zfQU_l1o,2564
+garf_executors/garf_pb2_grpc.py,sha256=w8D_r3wpj1ZZstkIFogY679-lSCcL2iZQ4QLO8IfToY,3359
+garf_executors/sql_executor.py,sha256=80WiuNBBWQz1y19LmWrzSk6auFFqh6YHBPTkFAGIhMs,4681
+garf_executors/telemetry.py,sha256=P75klGEoYgJ_-pR-izUIQ7B88ufskQ4vmW1rETg63Nc,747
+garf_executors/workflow.py,sha256=9Hkv0NgNyV5_xkkCTS6nsDVqtCmHfbqLQvdaIzFBrLU,2614
+garf_executors/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+garf_executors/entrypoints/cli.py,sha256=Kei2Tqdw1syPKhbeK5u-1G72hXgmo1arXmxio150jPE,6006
+garf_executors/entrypoints/grpc_server.py,sha256=zP9C-dStbElWkb0T_IcIAcBxmA9Wl4GTWytUcrC_7Xg,2296
+garf_executors/entrypoints/server.py,sha256=FbemRjrGDgpr51iAMXdvTXlP1OG7Rc5i5M55Prw0wXg,3473
+garf_executors/entrypoints/tracer.py,sha256=VylQMIXOsRLuT3UlFwjRy8GJiPUI6zohUXiGX_DcE4g,1912
+garf_executors/entrypoints/utils.py,sha256=5XiGR2IOxdzAOY0lEWUeUV7tIpKBGRnQaIwBYvzQB7c,4337
+garf_executors-0.2.3.dist-info/METADATA,sha256=96n_J13NrBFbfz2-fVYo0KlD5p9r7qO8AQ3R_K2V710,3055
+garf_executors-0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+garf_executors-0.2.3.dist-info/entry_points.txt,sha256=LskWNFIw8j0WJuI18-32OZrlASXAMg1XtrRYwsKBz2E,61
+garf_executors-0.2.3.dist-info/top_level.txt,sha256=sP4dCXOENPn1hDFAunjMV8Js4NND_KGeO_gQWuaT0EY,15
+garf_executors-0.2.3.dist-info/RECORD,,

garf_executors-0.1.4.dist-info/RECORD

@@ -1,20 +0,0 @@
-garf_executors/__init__.py,sha256=4BZv9zb3tjlpF4kQSdTj1L5IdR-BiNQwtejg5dPTTcY,1933
-garf_executors/api_executor.py,sha256=TxHtdnXjXjfBDU0z13yCulqF0XcEqAoOdVeGczdTSXs,3590
-garf_executors/bq_executor.py,sha256=LOKNitigaMk4U-UjBZTHy4vG092nw6suEbgo2rrHCTI,5002
-garf_executors/config.py,sha256=TqCzijm1PRvL4p-9Zl-kPkcC1SFKjhgTfKMJFmJW3fQ,1688
-garf_executors/exceptions.py,sha256=U_7Q2ZMOUf89gzZd2pw7y3g7i1NeByPPKfpZ3q7p3ZU,662
-garf_executors/execution_context.py,sha256=X4Wm_rE1mnnN2FuC_9bL05a8h8ko7qraeGY955ijNJc,2800
-garf_executors/executor.py,sha256=_Nj6CKgyhzwFOxneODDhV1bvLjrMEvIu93W8YF9-sXo,2481
-garf_executors/fetchers.py,sha256=HQqnMb0wlasVfXmAA7cnsd73POXPEGPxaC5mpEOnQk4,2443
-garf_executors/sql_executor.py,sha256=_4oVPZKTd3lrDE0SM6uQ_bl13Ay9uhQuD-PHO9247WM,3920
-garf_executors/telemetry.py,sha256=P75klGEoYgJ_-pR-izUIQ7B88ufskQ4vmW1rETg63Nc,747
-garf_executors/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-garf_executors/entrypoints/cli.py,sha256=Qbg10LLWHEMBjjsOfEMDZQtjWpUwh6WJKSnqiXOzF6A,4765
-garf_executors/entrypoints/server.py,sha256=b9blyBvN774RiTHUCZkfE5kNVnrTaANrETI4WMDHJeQ,3255
-garf_executors/entrypoints/tracer.py,sha256=A_nolmGuMT3wOZJsoPORjfdtPO2lXdbr6CZt5BW0RTY,1374
-garf_executors/entrypoints/utils.py,sha256=5XiGR2IOxdzAOY0lEWUeUV7tIpKBGRnQaIwBYvzQB7c,4337
-garf_executors-0.1.4.dist-info/METADATA,sha256=3Z0plyqxqwCKYOm2PlXIfvxGo0lAVkdIaLD0s0pgZzQ,2900
-garf_executors-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-garf_executors-0.1.4.dist-info/entry_points.txt,sha256=LskWNFIw8j0WJuI18-32OZrlASXAMg1XtrRYwsKBz2E,61
-garf_executors-0.1.4.dist-info/top_level.txt,sha256=sP4dCXOENPn1hDFAunjMV8Js4NND_KGeO_gQWuaT0EY,15
-garf_executors-0.1.4.dist-info/RECORD,,