garf-executors 1.0.2__py3-none-any.whl → 1.2.0__py3-none-any.whl

garf/executors/__init__.py

@@ -15,46 +15,11 @@
 
 from __future__ import annotations
 
-import importlib
-
-from garf.executors import executor, fetchers
 from garf.executors.api_executor import ApiExecutionContext, ApiQueryExecutor
-from garf.executors.telemetry import tracer
-
-
-@tracer.start_as_current_span('setup_executor')
-def setup_executor(
-  source: str,
-  fetcher_parameters: dict[str, str | int | bool],
-  enable_cache: bool = False,
-  cache_ttl_seconds: int = 3600,
-) -> type[executor.Executor]:
-  """Initializes executors based on a source and parameters."""
-  if source == 'bq':
-    bq_executor = importlib.import_module('garf.executors.bq_executor')
-    query_executor = bq_executor.BigQueryExecutor(**fetcher_parameters)
-  elif source == 'sqldb':
-    sql_executor = importlib.import_module('garf.executors.sql_executor')
-    query_executor = (
-      sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
-        fetcher_parameters.get('connection_string')
-      )
-    )
-  else:
-    concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    query_executor = ApiQueryExecutor(
-      fetcher=concrete_api_fetcher(
-        **fetcher_parameters,
-        enable_cache=enable_cache,
-        cache_ttl_seconds=cache_ttl_seconds,
-      )
-    )
-  return query_executor
-
 
 __all__ = [
   'ApiQueryExecutor',
   'ApiExecutionContext',
 ]
 
-__version__ = '1.0.2'
+__version__ = '1.2.0'

garf/executors/api_executor.py

@@ -21,8 +21,9 @@ GarfReport and saving it to local/remote storage.
 from __future__ import annotations
 
 import logging
+import pathlib
 
-from garf.core import report_fetcher
+from garf.core import report_fetcher, simulator
 from garf.executors import (
   exceptions,
   execution_context,
@@ -31,9 +32,17 @@ from garf.executors import (
   query_processor,
 )
 from garf.executors.telemetry import tracer
-from opentelemetry import trace
+from garf.io.writers import abs_writer
+from opentelemetry import metrics, trace
 
 logger = logging.getLogger(__name__)
+meter = metrics.get_meter('garf.executors')
+
+api_counter = meter.create_counter(
+  'garf_api_execute_total',
+  unit='1',
+  description='Counts number of API executions',
+)
 
 
 class ApiExecutionContext(execution_context.ExecutionContext):
@@ -49,13 +58,21 @@ class ApiQueryExecutor(executor.Executor):
       api_client: a client used for connecting to API.
   """
 
-  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
+  def __init__(
+    self,
+    fetcher: report_fetcher.ApiReportFetcher,
+    report_simulator: simulator.ApiReportSimulator | None = None,
+    writers: list[abs_writer.AbsWriter] | None = None,
+  ) -> None:
     """Initializes ApiQueryExecutor.
 
     Args:
-        fetcher: Instantiated report fetcher.
+      fetcher: Instantiated report fetcher.
+      report_simulator: Instantiated simulator.
     """
     self.fetcher = fetcher
+    self.simulator = report_simulator
+    self.writers = writers
     super().__init__(
       preprocessors=self.fetcher.preprocessors,
       postprocessors=self.fetcher.postprocessors,
@@ -100,6 +117,8 @@ class ApiQueryExecutor(executor.Executor):
     Raises:
       GarfExecutorError: When failed to execute query.
     """
+    if self.simulator:
+      return self.simulate(query=query, title=title, context=context)
     context = query_processor.process_gquery(context)
     span = trace.get_current_span()
     span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
@@ -107,15 +126,80 @@ class ApiQueryExecutor(executor.Executor):
       'api.client.class', self.fetcher.api_client.__class__.__name__
     )
     try:
-      span.set_attribute('query.title', title)
-      span.set_attribute('query.text', query)
       logger.debug('starting query %s', query)
+      title = pathlib.Path(title).name.split('.')[0]
+      api_counter.add(
+        1, {'api.client.class': self.fetcher.api_client.__class__.__name__}
+      )
       results = self.fetcher.fetch(
         query_specification=query,
         args=context.query_parameters,
+        title=title,
+        **context.fetcher_parameters,
+      )
+      writer_clients = self.writers or context.writer_clients
+      if not writer_clients:
+        logger.warning('No writers configured, skipping write operation')
+        return None
+      writing_results = []
+      for writer_client in writer_clients:
+        logger.debug(
+          'Start writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        result = writer_client.write(results, title)
+        logger.debug(
+          'Finish writing data for query %s via %s writer',
+          title,
+          type(writer_client),
+        )
+        writing_results.append(result)
+      logger.info('%s executed successfully', title)
+      # Return the last writer's result for backward compatibility
+      return writing_results[-1] if writing_results else None
+    except Exception as e:
+      logger.error('%s generated an exception: %s', title, str(e))
+      raise exceptions.GarfExecutorError(
+        '%s generated an exception: %s', title, str(e)
+      ) from e
+
+  @tracer.start_as_current_span('api.simulate')
+  def simulate(
+    self,
+    query: str,
+    title: str,
+    context: ApiExecutionContext,
+  ) -> str:
+    """Reads query, simulates results and stores them in a specified location.
+
+    Args:
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Returns:
+      Result of writing the report.
+
+    Raises:
+      GarfExecutorError: When failed to execute query.
+    """
+    context = query_processor.process_gquery(context)
+    span = trace.get_current_span()
+    span.set_attribute('simulator.class', self.simulator.__class__.__name__)
+    span.set_attribute(
+      'api.client.class', self.simulator.api_client.__class__.__name__
+    )
+    try:
+      logger.debug('starting query %s', query)
+      title = pathlib.Path(title).name.split('.')[0]
+      results = self.simulator.simulate(
+        query_specification=query,
+        args=context.query_parameters,
+        title=title,
         **context.fetcher_parameters,
       )
-      writer_clients = context.writer_clients
+      writer_clients = self.writers or context.writer_clients
       if not writer_clients:
         logger.warning('No writers configured, skipping write operation')
         return None

garf/executors/bq_executor.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 
 import contextlib
 import os
+import warnings
 
 try:
   from google.cloud import bigquery  # type: ignore
@@ -31,6 +32,7 @@ import logging
 from garf.core import query_editor, report
 from garf.executors import exceptions, execution_context, executor
 from garf.executors.telemetry import tracer
+from garf.io.writers import abs_writer
 from google.cloud import exceptions as google_cloud_exceptions
 from opentelemetry import trace
 
@@ -41,7 +43,7 @@ class BigQueryExecutorError(exceptions.GarfExecutorError):
   """Error when BigQueryExecutor fails to run query."""
 
 
-class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
+class BigQueryExecutor(executor.Executor):
   """Handles query execution in BigQuery.
 
   Attributes:
@@ -52,29 +54,42 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
 
   def __init__(
     self,
-    project_id: str | None = os.getenv('GOOGLE_CLOUD_PROJECT'),
+    project: str | None = os.getenv('GOOGLE_CLOUD_PROJECT'),
     location: str | None = None,
+    writers: list[abs_writer.AbsWriter] | None = None,
     **kwargs: str,
   ) -> None:
     """Initializes BigQueryExecutor.
 
     Args:
-        project_id: Google Cloud project id.
-        location: BigQuery dataset location.
+      project_id: Google Cloud project id.
+      location: BigQuery dataset location.
+      writers: Instantiated writers.
     """
-    if not project_id:
+    if not project and 'project_id' not in kwargs:
       raise BigQueryExecutorError(
-        'project_id is required. Either provide it as project_id parameter '
+        'project is required. Either provide it as project parameter '
         'or GOOGLE_CLOUD_PROJECT env variable.'
       )
-    self.project_id = project_id
+    if project_id := kwargs.get('project_id'):
+      warnings.warn(
+        "'project_id' parameter is deprecated. Please use 'project' instead.",
+        DeprecationWarning,
+        stacklevel=2,
+      )
+    self.project = project or project_id
     self.location = location
+    self.writers = writers
+    self._client = None
     super().__init__()
 
   @property
   def client(self) -> bigquery.Client:
-    """Instantiates bigquery client."""
-    return bigquery.Client(self.project_id)
+    """Instantiated BigQuery client."""
+    if not self._client:
+      with tracer.start_as_current_span('bq.create_client'):
+        self._client = bigquery.Client(self.project)
+    return self._client
 
   @tracer.start_as_current_span('bq.execute')
   def execute(
@@ -96,23 +111,23 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
       Report with data if query returns some data otherwise empty Report.
     """
     span = trace.get_current_span()
+    query_spec = (
+      query_editor.QuerySpecification(
+        text=query, title=title, args=context.query_parameters
+      )
+      .remove_comments()
+      .expand()
+    )
+    query_text = query_spec.query.text
+    title = query_spec.query.title
+    span.set_attribute('query.title', title)
+    span.set_attribute('query.text', query)
     logger.info('Executing script: %s', title)
-    query_text = self.replace_params_template(query, context.query_parameters)
+    # TODO: move to initialization
     self.create_datasets(context.query_parameters.macro)
-    job = self.client.query(query_text)
-    try:
-      result = job.result()
-    except google_cloud_exceptions.GoogleCloudError as e:
-      raise BigQueryExecutorError(
-        f'Failed to execute query {title}: Reason: {e}'
-      ) from e
-      logger.debug('%s launched successfully', title)
-    if result.total_rows:
-      results = report.GarfReport.from_pandas(result.to_dataframe())
-    else:
-      results = report.GarfReport()
-    if context.writer and results:
-      writer_clients = context.writer_clients
+    results = self._query(query_text, title)
+    if results and (self.writers or context.writer):
+      writer_clients = self.writers or context.writer_clients
       if not writer_clients:
         logger.warning('No writers configured, skipping write operation')
       else:
@@ -149,7 +164,7 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
     """
     if macros and (datasets := extract_datasets(macros)):
       for dataset in datasets:
-        dataset_id = f'{self.project_id}.{dataset}'
+        dataset_id = f'{self.project}.{dataset}'
         try:
           self.client.get_dataset(dataset_id)
         except google_cloud_exceptions.NotFound:
@@ -159,6 +174,19 @@ class BigQueryExecutor(executor.Executor, query_editor.TemplateProcessorMixin):
             self.client.create_dataset(bq_dataset, timeout=30)
             logger.info('Created new dataset %s', dataset_id)
 
+  def _query(self, query_text, title) -> report.GarfReport:
+    job = self.client.query(query_text)
+    try:
+      result = job.result()
+    except google_cloud_exceptions.GoogleCloudError as e:
+      raise BigQueryExecutorError(
+        f'Failed to execute query {title}: Reason: {e}'
+      ) from e
+      logger.debug('%s launched successfully', title)
+    if result.total_rows:
+      return report.GarfReport.from_pandas(result.to_dataframe())
+    return report.GarfReport()
+
 
 def extract_datasets(macros: dict | None) -> list[str]:
   """Finds dataset-related keys based on values in a dict.

garf/executors/entrypoints/cli.py

@@ -25,14 +25,19 @@ import pathlib
 import sys
 
 import garf.executors
-from garf.executors import config, exceptions, workflow
+from garf.executors import config, exceptions, setup
 from garf.executors.entrypoints import utils
-from garf.executors.entrypoints.tracer import initialize_tracer
+from garf.executors.entrypoints.tracer import (
+  initialize_meter,
+  initialize_tracer,
+)
 from garf.executors.telemetry import tracer
+from garf.executors.workflows import workflow, workflow_runner
 from garf.io import reader
 from opentelemetry import trace
 
 initialize_tracer()
+meter_provider = initialize_meter()
 
 
 @tracer.start_as_current_span('garf.entrypoints.cli')
@@ -53,6 +58,7 @@ def main():
   parser.add_argument(
     '--no-parallel-queries', dest='parallel_queries', action='store_false'
   )
+  parser.add_argument('--simulate', dest='simulate', action='store_true')
   parser.add_argument('--dry-run', dest='dry_run', action='store_true')
   parser.add_argument('-v', '--version', dest='version', action='store_true')
   parser.add_argument(
@@ -67,7 +73,12 @@ def main():
     default=3600,
     type=int,
   )
+  parser.add_argument('--workflow-skip', dest='workflow_skip', default=None)
+  parser.add_argument(
+    '--workflow-include', dest='workflow_include', default=None
+  )
   parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(simulate=False)
   parser.set_defaults(enable_cache=False)
   parser.set_defaults(dry_run=False)
   args, kwargs = parser.parse_known_args()
@@ -82,43 +93,38 @@ def main():
     loglevel=args.loglevel.upper(), logger_type=args.logger, name=args.log_name
   )
   reader_client = reader.create_reader(args.input)
+  param_types = ['source', 'macro', 'template']
+  outputs = args.output.split(',')
+  extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(kwargs)
+  source_parameters = extra_parameters.get('source', {})
+  writer_parameters = {}
+  for output in outputs:
+    writer_parameters.update(extra_parameters.get(output))
+
+  context = garf.executors.api_executor.ApiExecutionContext(
+    query_parameters={
+      'macro': extra_parameters.get('macro'),
+      'template': extra_parameters.get('template'),
+    },
+    writer=outputs,
+    writer_parameters=writer_parameters,
+    fetcher_parameters=source_parameters,
+  )
   if workflow_file := args.workflow:
     wf_parent = pathlib.Path.cwd() / pathlib.Path(workflow_file).parent
-    execution_workflow = workflow.Workflow.from_file(workflow_file)
-    for i, step in enumerate(execution_workflow.steps, 1):
-      with tracer.start_as_current_span(f'{i}-{step.fetcher}'):
-        query_executor = garf.executors.setup_executor(
-          source=step.fetcher,
-          fetcher_parameters=step.fetcher_parameters,
-          enable_cache=args.enable_cache,
-          cache_ttl_seconds=args.cache_ttl_seconds,
-        )
-        batch = {}
-        if not (queries := step.queries):
-          logger.error('Please provide one or more queries to run')
-          raise exceptions.GarfExecutorError(
-            'Please provide one or more queries to run'
-          )
-        for query in queries:
-          if isinstance(query, garf.executors.workflow.QueryPath):
-            query_path = wf_parent / pathlib.Path(query.path)
-            if not query_path.exists():
-              raise workflow.GarfWorkflowError(f'Query: {query_path} not found')
-            batch[query.path] = reader_client.read(query_path)
-          elif isinstance(query, garf.executors.workflow.QueryFolder):
-            query_path = wf_parent / pathlib.Path(query.folder)
-            if not query_path.exists():
-              raise workflow.GarfWorkflowError(
-                f'Folder: {query_path} not found'
-              )
-            for p in query_path.rglob('*'):
-              if p.suffix == '.sql':
-                batch[p.stem] = reader_client.read(p)
-          else:
-            batch[query.query.title] = query.query.text
-        query_executor.execute_batch(
-          batch, step.context, args.parallel_threshold
-        )
+    execution_workflow = workflow.Workflow.from_file(workflow_file, context)
+    workflow_skip = args.workflow_skip if args.workflow_skip else None
+    workflow_include = args.workflow_include if args.workflow_include else None
+    workflow_runner.WorkflowRunner(
+      execution_workflow=execution_workflow, wf_parent=wf_parent
+    ).run(
+      enable_cache=args.enable_cache,
+      cache_ttl_seconds=args.cache_ttl_seconds,
+      selected_aliases=workflow_include,
+      skipped_aliases=workflow_skip,
+      simulate=args.simulate,
+    )
+    meter_provider.shutdown()
     sys.exit()
 
   if not args.query:
@@ -132,31 +138,14 @@ def main():
       raise exceptions.GarfExecutorError(
         f'No execution context found for source {args.source} in {config_file}'
       )
-  else:
-    param_types = ['source', 'macro', 'template']
-    outputs = args.output.split(',')
-    extra_parameters = utils.ParamsParser([*param_types, *outputs]).parse(
-      kwargs
-    )
-    source_parameters = extra_parameters.get('source', {})
-    writer_parameters = {}
-    for output in outputs:
-      writer_parameters.update(extra_parameters.get(output))
-
-    context = garf.executors.api_executor.ApiExecutionContext(
-      query_parameters={
-        'macro': extra_parameters.get('macro'),
-        'template': extra_parameters.get('template'),
-      },
-      writer=outputs,
-      writer_parameters=writer_parameters,
-      fetcher_parameters=source_parameters,
-    )
-  query_executor = garf.executors.setup_executor(
+  query_executor = setup.setup_executor(
     source=args.source,
     fetcher_parameters=context.fetcher_parameters,
     enable_cache=args.enable_cache,
     cache_ttl_seconds=args.cache_ttl_seconds,
+    simulate=args.simulate,
+    writers=context.writer,
+    writer_parameters=context.writer_parameters,
   )
   batch = {query: reader_client.read(query) for query in args.query}
   if args.parallel_queries and len(args.query) > 1:
@@ -171,6 +160,7 @@ def main():
         query=reader_client.read(query), title=query, context=context
       )
   logging.shutdown()
+  meter_provider.shutdown()
 
 
 if __name__ == '__main__':

garf/executors/entrypoints/grpc_server.py

@@ -18,9 +18,8 @@ import argparse
 import logging
 from concurrent import futures
 
-import garf.executors
 import grpc
-from garf.executors import garf_pb2, garf_pb2_grpc
+from garf.executors import execution_context, garf_pb2, garf_pb2_grpc, setup
 from garf.executors.entrypoints.tracer import initialize_tracer
 from google.protobuf.json_format import MessageToDict
 from grpc_reflection.v1alpha import reflection
@@ -28,19 +27,34 @@ from grpc_reflection.v1alpha import reflection
 
 class GarfService(garf_pb2_grpc.GarfService):
   def Execute(self, request, context):
-    query_executor = garf.executors.setup_executor(
+    query_executor = setup.setup_executor(
       request.source, request.context.fetcher_parameters
     )
-    execution_context = garf.executors.execution_context.ExecutionContext(
-      **MessageToDict(request.context, preserving_proto_field_name=True)
-    )
     result = query_executor.execute(
       query=request.query,
       title=request.title,
-      context=execution_context,
+      context=execution_context.ExecutionContext(
+        **MessageToDict(request.context, preserving_proto_field_name=True)
+      ),
     )
     return garf_pb2.ExecuteResponse(results=[result])
 
+  def Fetch(self, request, context):
+    query_executor = setup.setup_executor(
+      request.source, request.context.fetcher_parameters
+    )
+    query_args = execution_context.ExecutionContext(
+      **MessageToDict(request.context, preserving_proto_field_name=True)
+    ).query_parameters
+    result = query_executor.fetcher.fetch(
+      query_specification=request.query,
+      title=request.title,
+      args=query_args,
+    )
+    return garf_pb2.FetchResponse(
+      columns=result.column_names, rows=result.to_list(row_type='dict')
+    )
+
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()

garf/executors/entrypoints/server.py

@@ -21,18 +21,55 @@ import garf.executors
 import pydantic
 import typer
 import uvicorn
-from garf.executors import exceptions
-from garf.executors.entrypoints.tracer import initialize_tracer
+from garf.executors import exceptions, setup
+from garf.executors.entrypoints import utils
+from garf.executors.entrypoints.tracer import (
+  initialize_meter,
+  initialize_tracer,
+)
+from garf.executors.workflows import workflow_runner
 from garf.io import reader
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing_extensions import Annotated
 
 initialize_tracer()
+initialize_meter()
 app = fastapi.FastAPI()
 FastAPIInstrumentor.instrument_app(app)
 typer_app = typer.Typer()
 
 
+class GarfSettings(BaseSettings):
+  """Specifies environmental variables for garf.
+
+  Ensure that mandatory variables are exposed via
+  export ENV_VARIABLE_NAME=VALUE.
+
+  Attributes:
+    loglevel: Level of logging.
+    log_name: Name of log.
+    logger_type: Type of logger.
+  """
+
+  model_config = SettingsConfigDict(env_prefix='garf_')
+
+  loglevel: str = 'INFO'
+  log_name: str = 'garf'
+  logger_type: str = 'local'
+
+
+class GarfDependencies:
+  def __init__(self) -> None:
+    """Initializes GarfDependencies."""
+    settings = GarfSettings()
+    self.logger = utils.init_logging(
+      loglevel=settings.loglevel,
+      logger_type=settings.logger_type,
+      name=settings.log_name,
+    )
+
+
 class ApiExecutorRequest(pydantic.BaseModel):
   """Request for executing a query.
 
@@ -81,14 +118,19 @@ async def version() -> str:
 
 
 @app.get('/api/fetchers')
-async def get_fetchers() -> list[str]:
+async def get_fetchers(
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> list[str]:
   """Shows all available API sources."""
   return list(garf.executors.fetchers.find_fetchers())
 
 
 @app.post('/api/execute')
-async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
-  query_executor = garf.executors.setup_executor(
+def execute(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
     request.source, request.context.fetcher_parameters
   )
   result = query_executor.execute(request.query, request.title, request.context)
@@ -96,8 +138,11 @@ async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
 
 
 @app.post('/api/execute:batch')
-def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
-  query_executor = garf.executors.setup_executor(
+def execute_batch(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
     request.source, request.context.fetcher_parameters
   )
   reader_client = reader.FileReader()
@@ -106,6 +151,18 @@ def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   return ApiExecutorResponse(results=results)
 
 
+@app.post('/api/execute:workflow')
+def execute_workflow(
+  workflow_file: str,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+  enable_cache: bool = False,
+  cache_ttl_seconds: int = 3600,
+) -> list[str]:
+  return workflow_runner.WorkflowRunner.from_file(workflow_file).run(
+    enable_cache=enable_cache, cache_ttl_seconds=cache_ttl_seconds
+  )
+
+
 @typer_app.command()
 def main(
   port: Annotated[int, typer.Option(help='Port to start the server')] = 8000,