garf-executors 0.0.11__py3-none-any.whl → 0.2.3__py3-none-any.whl

garf_executors/__init__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Executors to fetch data from various APIS and postprocess them."""
+"""Executors to fetch data from various APIs."""
 
 from __future__ import annotations
 
@@ -19,10 +19,15 @@ import importlib
 
 from garf_executors import executor, fetchers
 from garf_executors.api_executor import ApiExecutionContext, ApiQueryExecutor
+from garf_executors.telemetry import tracer
 
 
+@tracer.start_as_current_span('setup_executor')
 def setup_executor(
-  source: str, fetcher_parameters: dict[str, str]
+  source: str,
+  fetcher_parameters: dict[str, str | int | bool],
+  enable_cache: bool = False,
+  cache_ttl_seconds: int = 3600,
 ) -> type[executor.Executor]:
   """Initializes executors based on a source and parameters."""
   if source == 'bq':
@@ -38,7 +43,11 @@ def setup_executor(
   else:
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
     query_executor = ApiQueryExecutor(
-      concrete_api_fetcher(**fetcher_parameters)
+      fetcher=concrete_api_fetcher(
+        **fetcher_parameters,
+        enable_cache=enable_cache,
+        cache_ttl_seconds=cache_ttl_seconds,
+      )
     )
   return query_executor
 
@@ -48,4 +57,4 @@ __all__ = [
   'ApiExecutionContext',
 ]
 
-__version__ = '0.0.11'
+__version__ = '0.2.3'

garf_executors/api_executor.py

@@ -23,8 +23,10 @@ from __future__ import annotations
 import logging
 
 from garf_core import report_fetcher
+from opentelemetry import trace
 
 from garf_executors import exceptions, execution_context, executor, fetchers
+from garf_executors.telemetry import tracer
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +34,7 @@ logger = logging.getLogger(__name__)
 class ApiExecutionContext(execution_context.ExecutionContext):
   """Common context for executing one or more queries."""
 
-  writer: str = 'console'
+  writer: str | list[str] = 'console'
 
 
 class ApiQueryExecutor(executor.Executor):
@@ -49,34 +51,31 @@ class ApiQueryExecutor(executor.Executor):
         fetcher: Instantiated report fetcher.
     """
     self.fetcher = fetcher
+    super().__init__(
+      preprocessors=self.fetcher.preprocessors,
+      postprocessors=self.fetcher.postprocessors,
+    )
 
   @classmethod
   def from_fetcher_alias(
-    cls, source: str, fetcher_parameters: dict[str, str] | None = None
+    cls,
+    source: str,
+    fetcher_parameters: dict[str, str] | None = None,
+    enable_cache: bool = False,
+    cache_ttl_seconds: int = 3600,
   ) -> ApiQueryExecutor:
     if not fetcher_parameters:
       fetcher_parameters = {}
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    return ApiQueryExecutor(concrete_api_fetcher(**fetcher_parameters))
-
-  async def aexecute(
-    self,
-    query: str,
-    title: str,
-    context: ApiExecutionContext,
-  ) -> str:
-    """Performs query execution asynchronously.
-
-    Args:
-      query: Location of the query.
-      title: Name of the query.
-      context: Query execution context.
-
-    Returns:
-      Result of writing the report.
-    """
-    return await self.execute(query, context, title, context)
+    return ApiQueryExecutor(
+      fetcher=concrete_api_fetcher(
+        **fetcher_parameters,
+        enable_cache=enable_cache,
+        cache_ttl_seconds=cache_ttl_seconds,
+      )
+    )
 
+  @tracer.start_as_current_span('api.execute')
   def execute(
     self,
     query: str,
@@ -96,27 +95,41 @@ class ApiQueryExecutor(executor.Executor):
     Raises:
       GarfExecutorError: When failed to execute query.
     """
+    span = trace.get_current_span()
+    span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
+    span.set_attribute(
+      'api.client.class', self.fetcher.api_client.__class__.__name__
+    )
     try:
+      span.set_attribute('query.title', title)
+      span.set_attribute('query.text', query)
       logger.debug('starting query %s', query)
       results = self.fetcher.fetch(
         query_specification=query,
         args=context.query_parameters,
         **context.fetcher_parameters,
       )
-      writer_client = context.writer_client
-      logger.debug(
-        'Start writing data for query %s via %s writer',
-        title,
-        type(writer_client),
-      )
-      result = writer_client.write(results, title)
-      logger.debug(
-        'Finish writing data for query %s via %s writer',
-        title,
-        type(writer_client),
-      )
+      writer_clients = context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
+        return None
+      writing_results = []
+      for writer_client in writer_clients:
+        logger.debug(
+          'Start writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        result = writer_client.write(results, title)
+        logger.debug(
+          'Finish writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        writing_results.append(result)
       logger.info('%s executed successfully', title)
-      return result
+      # Return the last writer's result for backward compatibility
+      return writing_results[-1] if writing_results else None
     except Exception as e:
       logger.error('%s generated an exception: %s', title, str(e))
       raise exceptions.GarfExecutorError(

garf_executors/bq_executor.py

@@ -15,6 +15,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 
 try:
@@ -29,8 +30,10 @@ import logging
 
 from garf_core import query_editor, report
 from google.cloud import exceptions as google_cloud_exceptions
+from opentelemetry import trace
 
 from garf_executors import exceptions, execution_context, executor
+from garf_executors.telemetry import tracer
 
 logger = logging.getLogger(__name__)
 
@@ -66,12 +69,14 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
       )
     self.project_id = project_id
     self.location = location
+    super().__init__()
 
   @property
   def client(self) -> bigquery.Client:
     """Instantiates bigquery client."""
     return bigquery.Client(self.project_id)
 
+  @tracer.start_as_current_span('bq.execute')
   def execute(
     self,
     query: str,
@@ -90,35 +95,49 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
     Returns:
       Report with data if query returns some data otherwise empty Report.
     """
+    span = trace.get_current_span()
+    logger.info('Executing script: %s', title)
     query_text = self.replace_params_template(query, context.query_parameters)
     self.create_datasets(context.query_parameters.macro)
     job = self.client.query(query_text)
     try:
       result = job.result()
+    except google_cloud_exceptions.GoogleCloudError as e:
+      raise BigQueryExecutorError(
+        f'Failed to execute query {title}: Reason: {e}'
+      ) from e
       logger.debug('%s launched successfully', title)
-      if result.total_rows:
-        results = report.GarfReport.from_pandas(result.to_dataframe())
+    if result.total_rows:
+      results = report.GarfReport.from_pandas(result.to_dataframe())
+    else:
+      results = report.GarfReport()
+    if context.writer and results:
+      writer_clients = context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
       else:
-        results = report.GarfReport()
-      if context.writer and results:
-        writer_client = context.writer_client
-        logger.debug(
-          'Start writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        writing_result = writer_client.write(results, title)
-        logger.debug(
-          'Finish writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
+        writing_results = []
+        for writer_client in writer_clients:
+          logger.debug(
+            'Start writing data for query %s via %s writer',
+            title,
+            type(writer_client),
+          )
+          writing_result = writer_client.write(results, title)
+          logger.debug(
+            'Finish writing data for query %s via %s writer',
+            title,
+            type(writer_client),
+          )
+          writing_results.append(writing_result)
+        # Return the last writer's result for backward compatibility
         logger.info('%s executed successfully', title)
-        return writing_result
-      return results
-    except google_cloud_exceptions.GoogleCloudError as e:
-      raise BigQueryExecutorError(e) from e
+        return writing_results[-1] if writing_results else None
+    logger.info('%s executed successfully', title)
+    span.set_attribute('execute.num_results', len(results))
+    return results
 
+  @tracer.start_as_current_span('bq.create_datasets')
   def create_datasets(self, macros: dict | None) -> None:
     """Creates datasets in BQ based on values in a dict.
 
@@ -136,8 +155,9 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
         except google_cloud_exceptions.NotFound:
           bq_dataset = bigquery.Dataset(dataset_id)
           bq_dataset.location = self.location
-          self.client.create_dataset(bq_dataset, timeout=30)
-          logger.info('Created new dataset %s', dataset_id)
+          with contextlib.suppress(google_cloud_exceptions.Conflict):
+            self.client.create_dataset(bq_dataset, timeout=30)
+            logger.info('Created new dataset %s', dataset_id)
 
 
 def extract_datasets(macros: dict | None) -> list[str]:

garf_executors/config.py

@@ -47,5 +47,7 @@ class Config(pydantic.BaseModel):
   def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
     """Saves config to local or remote yaml file."""
     with smart_open.open(path, 'w', encoding='utf-8') as f:
-      yaml.dump(self.model_dump().get('sources'), f, encoding='utf-8')
+      yaml.dump(
+        self.model_dump(exclude_none=True).get('sources'), f, encoding='utf-8'
+      )
     return f'Config is saved to {str(path)}'

garf_executors/entrypoints/cli.py

@@ -20,24 +20,33 @@ storage.
 from __future__ import annotations
 
 import argparse
+import logging
 import sys
 
 from garf_io import reader
+from opentelemetry import trace
 
 import garf_executors
-from garf_executors import config, exceptions
+from garf_executors import config, exceptions, workflow
 from garf_executors.entrypoints import utils
+from garf_executors.entrypoints.tracer import initialize_tracer
+from garf_executors.telemetry import tracer
 
+initialize_tracer()
 
+
+@tracer.start_as_current_span('garf.entrypoints.cli')
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('query', nargs='*')
   parser.add_argument('-c', '--config', dest='config', default=None)
+  parser.add_argument('-w', '--workflow', dest='workflow', default=None)
   parser.add_argument('--source', dest='source', default=None)
   parser.add_argument('--output', dest='output', default='console')
   parser.add_argument('--input', dest='input', default='file')
   parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
   parser.add_argument('--logger', dest='logger', default='local')
+  parser.add_argument('--log-name', dest='log_name', default='garf')
   parser.add_argument(
     '--parallel-queries', dest='parallel_queries', action='store_true'
   )
@@ -47,61 +56,108 @@ def main():
   parser.add_argument('--dry-run', dest='dry_run', action='store_true')
   parser.add_argument('-v', '--version', dest='version', action='store_true')
   parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
+    '--parallel-threshold', dest='parallel_threshold', default=10, type=int
+  )
+  parser.add_argument(
+    '--enable-cache', dest='enable_cache', action='store_true'
+  )
+  parser.add_argument(
+    '--cache-ttl-seconds',
+    dest='cache_ttl_seconds',
+    default=3600,
+    type=int,
   )
   parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(enable_cache=False)
   parser.set_defaults(dry_run=False)
   args, kwargs = parser.parse_known_args()
 
+  span = trace.get_current_span()
+  command_args = ' '.join(sys.argv[1:])
+  span.set_attribute('cli.command', f'garf {command_args}')
   if args.version:
     print(garf_executors.__version__)
     sys.exit()
   logger = utils.init_logging(
-    loglevel=args.loglevel.upper(), logger_type=args.logger
+    loglevel=args.loglevel.upper(), logger_type=args.logger, name=args.log_name
   )
+  reader_client = reader.create_reader(args.input)
+  if workflow_file := args.workflow:
+    execution_workflow = workflow.Workflow.from_file(workflow_file)
+    for i, step in enumerate(execution_workflow.steps, 1):
+      with tracer.start_as_current_span(f'{i}-{step.fetcher}'):
+        query_executor = garf_executors.setup_executor(
+          source=step.fetcher,
+          fetcher_parameters=step.fetcher_parameters,
+          enable_cache=args.enable_cache,
+          cache_ttl_seconds=args.cache_ttl_seconds,
+        )
+        batch = {}
+        if not (queries := step.queries):
+          logger.error('Please provide one or more queries to run')
+          raise exceptions.GarfExecutorError(
+            'Please provide one or more queries to run'
+          )
+        for query in queries:
+          if isinstance(query, garf_executors.workflow.QueryPath):
+            batch[query.path] = reader_client.read(query.path)
+          else:
+            batch[query.query.title] = query.query.text
+        query_executor.execute_batch(
+          batch, step.context, args.parallel_threshold
+        )
+    sys.exit()
+
   if not args.query:
     logger.error('Please provide one or more queries to run')
     raise exceptions.GarfExecutorError(
       'Please provide one or more queries to run'
     )
-  reader_client = reader.create_reader(args.input)
   if config_file := args.config:
     execution_config = config.Config.from_file(config_file)
     if not (context := execution_config.sources.get(args.source)):
       raise exceptions.GarfExecutorError(
         f'No execution context found for source {args.source} in {config_file}'
       )
-    query_executor = garf_executors.setup_executor(
-      args.source, context.fetcher_parameters
-    )
-    batch = {query: reader_client.read(query) for query in args.query}
-    query_executor.execute_batch(batch, context, args.parallel_queries)
   else:
-    extra_parameters = utils.ParamsParser(
-      ['source', args.output, 'macro', 'template']
-    ).parse(kwargs)
+    param_types = ['source', 'macro', 'template']
+    outputs = args.output.split(',')
+    extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(
+      kwargs
+    )
     source_parameters = extra_parameters.get('source', {})
+    writer_parameters = {}
+    for output in outputs:
+      writer_parameters.update(extra_parameters.get(output))
 
     context = garf_executors.api_executor.ApiExecutionContext(
       query_parameters={
         'macro': extra_parameters.get('macro'),
         'template': extra_parameters.get('template'),
       },
-      writer=args.output,
-      writer_parameters=extra_parameters.get(args.output),
+      writer=outputs,
+      writer_parameters=writer_parameters,
       fetcher_parameters=source_parameters,
     )
-    query_executor = garf_executors.setup_executor(
-      args.source, context.fetcher_parameters
-    )
-    if args.parallel_queries:
-      logger.info('Running queries in parallel')
-      batch = {query: reader_client.read(query) for query in args.query}
-      query_executor.execute_batch(batch, context, args.parallel_queries)
-    else:
+  query_executor = garf_executors.setup_executor(
+    source=args.source,
+    fetcher_parameters=context.fetcher_parameters,
+    enable_cache=args.enable_cache,
+    cache_ttl_seconds=args.cache_ttl_seconds,
+  )
+  batch = {query: reader_client.read(query) for query in args.query}
+  if args.parallel_queries and len(args.query) > 1:
+    logger.info('Running queries in parallel')
+    batch = {query: reader_client.read(query) for query in args.query}
+    query_executor.execute_batch(batch, context, args.parallel_threshold)
+  else:
+    if len(args.query) > 1:
       logger.info('Running queries sequentially')
-      for query in args.query:
-        query_executor.execute(reader_client.read(query), query, context)
+    for query in args.query:
+      query_executor.execute(
+        query=reader_client.read(query), title=query, context=context
+      )
+  logging.shutdown()
 
 
 if __name__ == '__main__':

garf_executors/entrypoints/grpc_server.py

@@ -0,0 +1,68 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""gRPC endpoint for garf."""
+
+import argparse
+import logging
+from concurrent import futures
+
+import grpc
+from google.protobuf.json_format import MessageToDict
+from grpc_reflection.v1alpha import reflection
+
+import garf_executors
+from garf_executors import garf_pb2, garf_pb2_grpc
+from garf_executors.entrypoints.tracer import initialize_tracer
+
+
+class GarfService(garf_pb2_grpc.GarfService):
+  def Execute(self, request, context):
+    query_executor = garf_executors.setup_executor(
+      request.source, request.context.fetcher_parameters
+    )
+    execution_context = garf_executors.execution_context.ExecutionContext(
+      **MessageToDict(request.context, preserving_proto_field_name=True)
+    )
+    result = query_executor.execute(
+      query=request.query,
+      title=request.title,
+      context=execution_context,
+    )
+    return garf_pb2.ExecuteResponse(results=[result])
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--port', dest='port', default=50051, type=int)
+  parser.add_argument(
+    '--parallel-threshold', dest='parallel_threshold', default=10, type=int
+  )
+  args, _ = parser.parse_known_args()
+  initialize_tracer()
+  server = grpc.server(
+    futures.ThreadPoolExecutor(max_workers=args.parallel_threshold)
+  )
+
+  service = GarfService()
+  garf_pb2_grpc.add_GarfServiceServicer_to_server(service, server)
+  SERVICE_NAMES = (
+    garf_pb2.DESCRIPTOR.services_by_name['GarfService'].full_name,
+    reflection.SERVICE_NAME,
+  )
+  reflection.enable_server_reflection(SERVICE_NAMES, server)
+  server.add_insecure_port(f'[::]:{args.port}')
+  server.start()
+  logging.info('Garf service started, listening on port %d', 50051)
+  server.wait_for_termination()

garf_executors/entrypoints/server.py

@@ -18,11 +18,20 @@ from typing import Optional, Union
 
 import fastapi
 import pydantic
+import typer
 import uvicorn
 from garf_io import reader
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from typing_extensions import Annotated
 
 import garf_executors
 from garf_executors import exceptions
+from garf_executors.entrypoints.tracer import initialize_tracer
+
+initialize_tracer()
+app = fastapi.FastAPI()
+FastAPIInstrumentor.instrument_app(app)
+typer_app = typer.Typer()
 
 
 class ApiExecutorRequest(pydantic.BaseModel):
@@ -40,7 +49,7 @@ class ApiExecutorRequest(pydantic.BaseModel):
   title: Optional[str] = None
   query: Optional[str] = None
   query_path: Optional[Union[str, list[str]]] = None
-  context: garf_executors.ApiExecutionContext
+  context: garf_executors.api_executor.ApiExecutionContext
 
   @pydantic.model_validator(mode='after')
   def check_query_specified(self):
@@ -67,10 +76,18 @@ class ApiExecutorResponse(pydantic.BaseModel):
   results: list[str]
 
 
-router = fastapi.APIRouter(prefix='/api')
+@app.get('/api/version')
+async def version() -> str:
+  return garf_executors.__version__
+
+
+@app.get('/api/fetchers')
+async def get_fetchers() -> list[str]:
+  """Shows all available API sources."""
+  return list(garf_executors.fetchers.find_fetchers())
 
 
-@router.post('/execute')
+@app.post('/api/execute')
 async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
   query_executor = garf_executors.setup_executor(
     request.source, request.context.fetcher_parameters
@@ -79,8 +96,8 @@ async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
   return ApiExecutorResponse(results=[result])
 
 
-@router.post('/execute:batch')
-async def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
+@app.post('/api/execute:batch')
+def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   query_executor = garf_executors.setup_executor(
     request.source, request.context.fetcher_parameters
   )
@@ -90,7 +107,12 @@ async def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   return ApiExecutorResponse(results=results)
 
 
+@typer_app.command()
+def main(
+  port: Annotated[int, typer.Option(help='Port to start the server')] = 8000,
+):
+  uvicorn.run(app, port=port)
+
+
 if __name__ == '__main__':
-  app = fastapi.FastAPI()
-  app.include_router(router)
-  uvicorn.run(app)
+  typer_app()

garf_executors/entrypoints/tracer.py

@@ -0,0 +1,57 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+  OTLPSpanExporter,
+)
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+  BatchSpanProcessor,
+)
+
+DEFAULT_SERVICE_NAME = 'garf'
+
+
+def initialize_tracer():
+  resource = Resource.create(
+    {'service.name': os.getenv('OTLP_SERVICE_NAME', DEFAULT_SERVICE_NAME)}
+  )
+
+  tracer_provider = TracerProvider(resource=resource)
+
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    if gcp_project_id := os.getenv('OTEL_EXPORTER_GCP_PROJECT_ID'):
+      try:
+        from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+      except ImportError as e:
+        raise ImportError(
+          'Please install garf_executors with GCP support '
+          '- `pip install garf_executors[gcp]`'
+        ) from e
+
+      cloud_span_processor = BatchSpanProcessor(
+        CloudTraceSpanExporter(project_id=gcp_project_id)
+      )
+      tracer_provider.add_span_processor(cloud_span_processor)
+    else:
+      otlp_processor = BatchSpanProcessor(
+        OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
+      )
+      tracer_provider.add_span_processor(otlp_processor)
+
+  trace.set_tracer_provider(tracer_provider)