garf-executors 1.0.2__py3-none-any.whl → 1.1.3__py3-none-any.whl

garf/executors/__init__.py

@@ -15,46 +15,11 @@
 
 from __future__ import annotations
 
-import importlib
-
-from garf.executors import executor, fetchers
 from garf.executors.api_executor import ApiExecutionContext, ApiQueryExecutor
-from garf.executors.telemetry import tracer
-
-
-@tracer.start_as_current_span('setup_executor')
-def setup_executor(
-  source: str,
-  fetcher_parameters: dict[str, str | int | bool],
-  enable_cache: bool = False,
-  cache_ttl_seconds: int = 3600,
-) -> type[executor.Executor]:
-  """Initializes executors based on a source and parameters."""
-  if source == 'bq':
-    bq_executor = importlib.import_module('garf.executors.bq_executor')
-    query_executor = bq_executor.BigQueryExecutor(**fetcher_parameters)
-  elif source == 'sqldb':
-    sql_executor = importlib.import_module('garf.executors.sql_executor')
-    query_executor = (
-      sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
-        fetcher_parameters.get('connection_string')
-      )
-    )
-  else:
-    concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    query_executor = ApiQueryExecutor(
-      fetcher=concrete_api_fetcher(
-        **fetcher_parameters,
-        enable_cache=enable_cache,
-        cache_ttl_seconds=cache_ttl_seconds,
-      )
-    )
-  return query_executor
-
 
 __all__ = [
   'ApiQueryExecutor',
   'ApiExecutionContext',
 ]
 
-__version__ = '1.0.2'
+__version__ = '1.1.3'

garf/executors/api_executor.py

@@ -21,8 +21,9 @@ GarfReport and saving it to local/remote storage.
 from __future__ import annotations
 
 import logging
+import pathlib
 
-from garf.core import report_fetcher
+from garf.core import report_fetcher, simulator
 from garf.executors import (
   exceptions,
   execution_context,
@@ -31,9 +32,16 @@ from garf.executors import (
   query_processor,
 )
 from garf.executors.telemetry import tracer
-from opentelemetry import trace
+from opentelemetry import metrics, trace
 
 logger = logging.getLogger(__name__)
+meter = metrics.get_meter('garf.executors')
+
+api_counter = meter.create_counter(
+  'garf_api_execute_total',
+  unit='1',
+  description='Counts number of API executions',
+)
 
 
 class ApiExecutionContext(execution_context.ExecutionContext):
@@ -49,13 +57,19 @@ class ApiQueryExecutor(executor.Executor):
       api_client: a client used for connecting to API.
   """
 
-  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
+  def __init__(
+    self,
+    fetcher: report_fetcher.ApiReportFetcher,
+    report_simulator: simulator.ApiReportSimulator | None = None,
+  ) -> None:
     """Initializes ApiQueryExecutor.
 
     Args:
-        fetcher: Instantiated report fetcher.
+      fetcher: Instantiated report fetcher.
+      report_simulator: Instantiated simulator.
     """
     self.fetcher = fetcher
+    self.simulator = report_simulator
     super().__init__(
       preprocessors=self.fetcher.preprocessors,
       postprocessors=self.fetcher.postprocessors,
@@ -100,6 +114,8 @@ class ApiQueryExecutor(executor.Executor):
     Raises:
       GarfExecutorError: When failed to execute query.
     """
+    if self.simulator:
+      return self.simulate(query=query, title=title, context=context)
     context = query_processor.process_gquery(context)
     span = trace.get_current_span()
     span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
@@ -110,9 +126,78 @@ class ApiQueryExecutor(executor.Executor):
       span.set_attribute('query.title', title)
       span.set_attribute('query.text', query)
       logger.debug('starting query %s', query)
+      title = pathlib.Path(title).name.split('.')[0]
+      api_counter.add(
+        1, {'api.client.class': self.fetcher.api_client.__class__.__name__}
+      )
       results = self.fetcher.fetch(
         query_specification=query,
         args=context.query_parameters,
+        title=title,
+        **context.fetcher_parameters,
+      )
+      writer_clients = context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
+        return None
+      writing_results = []
+      for writer_client in writer_clients:
+        logger.debug(
+          'Start writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        result = writer_client.write(results, title)
+        logger.debug(
+          'Finish writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        writing_results.append(result)
+      logger.info('%s executed successfully', title)
+      # Return the last writer's result for backward compatibility
+      return writing_results[-1] if writing_results else None
+    except Exception as e:
+      logger.error('%s generated an exception: %s', title, str(e))
+      raise exceptions.GarfExecutorError(
+        '%s generated an exception: %s', title, str(e)
+      ) from e
+
+  @tracer.start_as_current_span('api.simulate')
+  def simulate(
+    self,
+    query: str,
+    title: str,
+    context: ApiExecutionContext,
+  ) -> str:
+    """Reads query, simulates results and stores them in a specified location.
+
+    Args:
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Returns:
+      Result of writing the report.
+
+    Raises:
+      GarfExecutorError: When failed to execute query.
+    """
+    context = query_processor.process_gquery(context)
+    span = trace.get_current_span()
+    span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
+    span.set_attribute(
+      'api.client.class', self.fetcher.api_client.__class__.__name__
+    )
+    try:
+      span.set_attribute('query.title', title)
+      span.set_attribute('query.text', query)
+      logger.debug('starting query %s', query)
+      title = pathlib.Path(title).name.split('.')[0]
+      results = self.simulator.simulate(
+        query_specification=query,
+        args=context.query_parameters,
+        title=title,
         **context.fetcher_parameters,
       )
       writer_clients = context.writer_clients

garf/executors/bq_executor.py

@@ -96,6 +96,8 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
       Report with data if query returns some data otherwise empty Report.
     """
     span = trace.get_current_span()
+    span.set_attribute('query.title', title)
+    span.set_attribute('query.text', query)
     logger.info('Executing script: %s', title)
     query_text = self.replace_params_template(query, context.query_parameters)
     self.create_datasets(context.query_parameters.macro)

garf/executors/entrypoints/cli.py

@@ -25,14 +25,19 @@ import pathlib
 import sys
 
 import garf.executors
-from garf.executors import config, exceptions, workflow
+from garf.executors import config, exceptions, setup
 from garf.executors.entrypoints import utils
-from garf.executors.entrypoints.tracer import initialize_tracer
+from garf.executors.entrypoints.tracer import (
+  initialize_meter,
+  initialize_tracer,
+)
 from garf.executors.telemetry import tracer
+from garf.executors.workflows import workflow, workflow_runner
 from garf.io import reader
 from opentelemetry import trace
 
 initialize_tracer()
+meter_provider = initialize_meter()
 
 
 @tracer.start_as_current_span('garf.entrypoints.cli')
@@ -53,6 +58,7 @@ def main():
   parser.add_argument(
     '--no-parallel-queries', dest='parallel_queries', action='store_false'
   )
+  parser.add_argument('--simulate', dest='simulate', action='store_true')
   parser.add_argument('--dry-run', dest='dry_run', action='store_true')
   parser.add_argument('-v', '--version', dest='version', action='store_true')
   parser.add_argument(
@@ -67,7 +73,12 @@ def main():
     default=3600,
     type=int,
   )
+  parser.add_argument('--workflow-skip', dest='workflow_skip', default=None)
+  parser.add_argument(
+    '--workflow-include', dest='workflow_include', default=None
+  )
   parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(simulate=False)
   parser.set_defaults(enable_cache=False)
   parser.set_defaults(dry_run=False)
   args, kwargs = parser.parse_known_args()
@@ -82,43 +93,37 @@ def main():
     loglevel=args.loglevel.upper(), logger_type=args.logger, name=args.log_name
   )
   reader_client = reader.create_reader(args.input)
+  param_types = ['source', 'macro', 'template']
+  outputs = args.output.split(',')
+  extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(kwargs)
+  source_parameters = extra_parameters.get('source', {})
+  writer_parameters = {}
+  for output in outputs:
+    writer_parameters.update(extra_parameters.get(output))
+
+  context = garf.executors.api_executor.ApiExecutionContext(
+    query_parameters={
+      'macro': extra_parameters.get('macro'),
+      'template': extra_parameters.get('template'),
+    },
+    writer=outputs,
+    writer_parameters=writer_parameters,
+    fetcher_parameters=source_parameters,
+  )
   if workflow_file := args.workflow:
     wf_parent = pathlib.Path.cwd() / pathlib.Path(workflow_file).parent
-    execution_workflow = workflow.Workflow.from_file(workflow_file)
-    for i, step in enumerate(execution_workflow.steps, 1):
-      with tracer.start_as_current_span(f'{i}-{step.fetcher}'):
-        query_executor = garf.executors.setup_executor(
-          source=step.fetcher,
-          fetcher_parameters=step.fetcher_parameters,
-          enable_cache=args.enable_cache,
-          cache_ttl_seconds=args.cache_ttl_seconds,
-        )
-        batch = {}
-        if not (queries := step.queries):
-          logger.error('Please provide one or more queries to run')
-          raise exceptions.GarfExecutorError(
-            'Please provide one or more queries to run'
-          )
-        for query in queries:
-          if isinstance(query, garf.executors.workflow.QueryPath):
-            query_path = wf_parent / pathlib.Path(query.path)
-            if not query_path.exists():
-              raise workflow.GarfWorkflowError(f'Query: {query_path} not found')
-            batch[query.path] = reader_client.read(query_path)
-          elif isinstance(query, garf.executors.workflow.QueryFolder):
-            query_path = wf_parent / pathlib.Path(query.folder)
-            if not query_path.exists():
-              raise workflow.GarfWorkflowError(
-                f'Folder: {query_path} not found'
-              )
-            for p in query_path.rglob('*'):
-              if p.suffix == '.sql':
-                batch[p.stem] = reader_client.read(p)
-          else:
-            batch[query.query.title] = query.query.text
-        query_executor.execute_batch(
-          batch, step.context, args.parallel_threshold
-        )
+    execution_workflow = workflow.Workflow.from_file(workflow_file, context)
+    workflow_skip = args.workflow_skip if args.workflow_skip else None
+    workflow_include = args.workflow_include if args.workflow_include else None
+    workflow_runner.WorkflowRunner(
+      execution_workflow=execution_workflow, wf_parent=wf_parent
+    ).run(
+      enable_cache=args.enable_cache,
+      cache_ttl_seconds=args.cache_ttl_seconds,
+      selected_aliases=workflow_include,
+      skipped_aliases=workflow_skip,
+    )
+    meter_provider.shutdown()
     sys.exit()
 
   if not args.query:
@@ -132,31 +137,12 @@ def main():
       raise exceptions.GarfExecutorError(
         f'No execution context found for source {args.source} in {config_file}'
       )
-  else:
-    param_types = ['source', 'macro', 'template']
-    outputs = args.output.split(',')
-    extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(
-      kwargs
-    )
-    source_parameters = extra_parameters.get('source', {})
-    writer_parameters = {}
-    for output in outputs:
-      writer_parameters.update(extra_parameters.get(output))
-
-    context = garf.executors.api_executor.ApiExecutionContext(
-      query_parameters={
-        'macro': extra_parameters.get('macro'),
-        'template': extra_parameters.get('template'),
-      },
-      writer=outputs,
-      writer_parameters=writer_parameters,
-      fetcher_parameters=source_parameters,
-    )
-  query_executor = garf.executors.setup_executor(
+  query_executor = setup.setup_executor(
     source=args.source,
     fetcher_parameters=context.fetcher_parameters,
     enable_cache=args.enable_cache,
     cache_ttl_seconds=args.cache_ttl_seconds,
+    simulate=args.simulate,
   )
   batch = {query: reader_client.read(query) for query in args.query}
   if args.parallel_queries and len(args.query) > 1:
@@ -171,6 +157,7 @@ def main():
         query=reader_client.read(query), title=query, context=context
       )
   logging.shutdown()
+  meter_provider.shutdown()
 
 
 if __name__ == '__main__':

garf/executors/entrypoints/grpc_server.py

@@ -18,9 +18,8 @@ import argparse
 import logging
 from concurrent import futures
 
-import garf.executors
 import grpc
-from garf.executors import garf_pb2, garf_pb2_grpc
+from garf.executors import execution_context, garf_pb2, garf_pb2_grpc, setup
 from garf.executors.entrypoints.tracer import initialize_tracer
 from google.protobuf.json_format import MessageToDict
 from grpc_reflection.v1alpha import reflection
@@ -28,19 +27,34 @@ from grpc_reflection.v1alpha import reflection
 
 class GarfService(garf_pb2_grpc.GarfService):
   def Execute(self, request, context):
-    query_executor = garf.executors.setup_executor(
+    query_executor = setup.setup_executor(
       request.source, request.context.fetcher_parameters
     )
-    execution_context = garf.executors.execution_context.ExecutionContext(
-      **MessageToDict(request.context, preserving_proto_field_name=True)
-    )
     result = query_executor.execute(
       query=request.query,
       title=request.title,
-      context=execution_context,
+      context=execution_context.ExecutionContext(
+        **MessageToDict(request.context, preserving_proto_field_name=True)
+      ),
     )
     return garf_pb2.ExecuteResponse(results=[result])
 
+  def Fetch(self, request, context):
+    query_executor = setup.setup_executor(
+      request.source, request.context.fetcher_parameters
+    )
+    query_args = execution_context.ExecutionContext(
+      **MessageToDict(request.context, preserving_proto_field_name=True)
+    ).query_parameters
+    result = query_executor.fetcher.fetch(
+      query_specification=request.query,
+      title=request.title,
+      args=query_args,
+    )
+    return garf_pb2.FetchResponse(
+      columns=result.column_names, rows=result.to_list(row_type='dict')
+    )
+
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()

garf/executors/entrypoints/server.py

@@ -21,18 +21,55 @@ import garf.executors
 import pydantic
 import typer
 import uvicorn
-from garf.executors import exceptions
-from garf.executors.entrypoints.tracer import initialize_tracer
+from garf.executors import exceptions, setup
+from garf.executors.entrypoints import utils
+from garf.executors.entrypoints.tracer import (
+  initialize_meter,
+  initialize_tracer,
+)
+from garf.executors.workflows import workflow_runner
 from garf.io import reader
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing_extensions import Annotated
 
 initialize_tracer()
+initialize_meter()
 app = fastapi.FastAPI()
 FastAPIInstrumentor.instrument_app(app)
 typer_app = typer.Typer()
 
 
+class GarfSettings(BaseSettings):
+  """Specifies environmental variables for garf.
+
+  Ensure that mandatory variables are exposed via
+  export ENV_VARIABLE_NAME=VALUE.
+
+  Attributes:
+    loglevel: Level of logging.
+    log_name: Name of log.
+    logger_type: Type of logger.
+  """
+
+  model_config = SettingsConfigDict(env_prefix='garf_')
+
+  loglevel: str = 'INFO'
+  log_name: str = 'garf'
+  logger_type: str = 'local'
+
+
+class GarfDependencies:
+  def __init__(self) -> None:
+    """Initializes GarfDependencies."""
+    settings = GarfSettings()
+    self.logger = utils.init_logging(
+      loglevel=settings.loglevel,
+      logger_type=settings.logger_type,
+      name=settings.log_name,
+    )
+
+
 class ApiExecutorRequest(pydantic.BaseModel):
   """Request for executing a query.
 
@@ -81,14 +118,19 @@ async def version() -> str:
 
 
 @app.get('/api/fetchers')
-async def get_fetchers() -> list[str]:
+async def get_fetchers(
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> list[str]:
   """Shows all available API sources."""
   return list(garf.executors.fetchers.find_fetchers())
 
 
 @app.post('/api/execute')
-async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
-  query_executor = garf.executors.setup_executor(
+def execute(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
     request.source, request.context.fetcher_parameters
   )
   result = query_executor.execute(request.query, request.title, request.context)
@@ -96,8 +138,11 @@ async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
 
 
 @app.post('/api/execute:batch')
-def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
-  query_executor = garf.executors.setup_executor(
+def execute_batch(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
     request.source, request.context.fetcher_parameters
   )
   reader_client = reader.FileReader()
@@ -106,6 +151,18 @@ def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   return ApiExecutorResponse(results=results)
 
 
+@app.post('/api/execute:workflow')
+def execute_workflow(
+  workflow_file: str,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+  enable_cache: bool = False,
+  cache_ttl_seconds: int = 3600,
+) -> list[str]:
+  return workflow_runner.WorkflowRunner.from_file(workflow_file).run(
+    enable_cache=enable_cache, cache_ttl_seconds=cache_ttl_seconds
+  )
+
+
 @typer_app.command()
 def main(
   port: Annotated[int, typer.Option(help='Port to start the server')] = 8000,

garf/executors/entrypoints/tracer.py

@@ -14,15 +14,20 @@
 
 import os
 
-from opentelemetry import trace
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+  OTLPMetricExporter,
+)
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
   OTLPSpanExporter,
 )
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+  PeriodicExportingMetricReader,
+)
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import (
-  BatchSpanProcessor,
-)
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
 DEFAULT_SERVICE_NAME = 'garf'
 
@@ -55,3 +60,23 @@ def initialize_tracer():
       tracer_provider.add_span_processor(otlp_processor)
 
   trace.set_tracer_provider(tracer_provider)
+
+
+def initialize_meter():
+  resource = Resource.create(
+    {'service.name': os.getenv('OTLP_SERVICE_NAME', DEFAULT_SERVICE_NAME)}
+  )
+  meter_provider = MeterProvider(resource=resource)
+
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    otlp_metric_exporter = OTLPMetricExporter(
+      endpoint=f'{otel_endpoint}/v1/metrics'
+    )
+    metric_reader = PeriodicExportingMetricReader(otlp_metric_exporter)
+    meter_provider = MeterProvider(
+      resource=resource, metric_readers=[metric_reader]
+    )
+  else:
+    meter_provider = MeterProvider(resource=resource)
+  metrics.set_meter_provider(meter_provider)
+  return meter_provider

garf/executors/fetchers.py

@@ -17,7 +17,7 @@ import logging
 import sys
 from importlib.metadata import entry_points
 
-from garf.core import report_fetcher
+from garf.core import report_fetcher, simulator
 from garf.executors.telemetry import tracer
 
 logger = logging.getLogger(name='garf.executors.fetchers')
@@ -31,6 +31,14 @@ def find_fetchers() -> set[str]:
   return set()
 
 
+@tracer.start_as_current_span('find_simulators')
+def find_simulators() -> set[str]:
+  """Identifiers all available report simulators."""
+  if entrypoints := _get_entrypoints('garf_simulator'):
+    return {simulator.name for simulator in entrypoints}
+  return set()
+
+
 @tracer.start_as_current_span('get_report_fetcher')
 def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
   """Loads report fetcher for a given source.
@@ -57,7 +65,10 @@ def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
           if inspect.isclass(obj) and issubclass(
             obj, report_fetcher.ApiReportFetcher
           ):
-            return getattr(fetcher_module, name)
+            if not hasattr(obj, 'alias'):
+              return getattr(fetcher_module, name)
+            if obj.alias == fetcher.name:
+              return getattr(fetcher_module, name)
       except ModuleNotFoundError as e:
         raise report_fetcher.ApiReportFetcherError(
           f'Failed to load fetcher for source {source}, reason: {e}'
@@ -67,6 +78,45 @@ def get_report_fetcher(source: str) -> type[report_fetcher.ApiReportFetcher]:
   )
 
 
+@tracer.start_as_current_span('get_report_simulator')
+def get_report_simulator(source: str) -> type[simulator.ApiReportSimulator]:
+  """Loads report simulator for a given source.
+
+  Args:
+    source: Alias for a source associated with a simulator.
+
+  Returns:
+    Class for a found report simulator.
+
+  Raises:
+    GarfApiReportSimulatorError: When simulator cannot be loaded.
+    MissingApiReportSimulatorError: When simulator not found.
+  """
+  if source not in find_simulators():
+    raise simulator.MissingApiReportSimulatorError(source)
+  for sim in _get_entrypoints('garf_simulator'):
+    if sim.name == source:
+      try:
+        with tracer.start_as_current_span('load_simulator_module') as span:
+          simulator_module = sim.load()
+          span.set_attribute('loaded_module', simulator_module.__name__)
+        for name, obj in inspect.getmembers(simulator_module):
+          if inspect.isclass(obj) and issubclass(
+            obj, simulator.ApiReportSimulator
+          ):
+            if not hasattr(obj, 'alias'):
+              return getattr(simulator_module, name)
+            if obj.alias == sim.name:
+              return getattr(simulator_module, name)
+      except ModuleNotFoundError as e:
+        raise simulator.GarfApiReportSimulatorError(
+          f'Failed to load simulator for source {source}, reason: {e}'
+        )
+  raise simulator.GarfApiReportSimulatorError(
+    f'No simulator available for the source "{source}"'
+  )
+
+
 def _get_entrypoints(group='garf'):
   if sys.version_info.major == 3 and sys.version_info.minor == 9:
     try: