garf-executors 0.1.7__tar.gz → 0.2.0__tar.gz

{garf_executors-0.1.7 → garf_executors-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.1.7
+Version: 0.2.0
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_executors-0.1.7 → garf_executors-0.2.0}/garf_executors/__init__.py

@@ -43,7 +43,7 @@ def setup_executor(
   else:
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
     query_executor = ApiQueryExecutor(
-      concrete_api_fetcher(
+      fetcher=concrete_api_fetcher(
         **fetcher_parameters,
         enable_cache=enable_cache,
         cache_ttl_seconds=cache_ttl_seconds,
@@ -57,4 +57,4 @@ __all__ = [
   'ApiExecutionContext',
 ]
 
-__version__ = '0.1.7'
+__version__ = '0.2.0'

{garf_executors-0.1.7 → garf_executors-0.2.0}/garf_executors/api_executor.py

@@ -51,6 +51,10 @@ class ApiQueryExecutor(executor.Executor):
         fetcher: Instantiated report fetcher.
     """
     self.fetcher = fetcher
+    super().__init__(
+      preprocessors=self.fetcher.preprocessors,
+      postprocessors=self.fetcher.postprocessors,
+    )
 
   @classmethod
   def from_fetcher_alias(
@@ -59,7 +63,7 @@ class ApiQueryExecutor(executor.Executor):
     if not fetcher_parameters:
       fetcher_parameters = {}
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    return ApiQueryExecutor(concrete_api_fetcher(**fetcher_parameters))
+    return ApiQueryExecutor(fetcher=concrete_api_fetcher(**fetcher_parameters))
 
   @tracer.start_as_current_span('api.execute')
   def execute(

{garf_executors-0.1.7 → garf_executors-0.2.0}/garf_executors/entrypoints/cli.py

@@ -88,14 +88,6 @@ def main():
       raise exceptions.GarfExecutorError(
         f'No execution context found for source {args.source} in {config_file}'
       )
-    query_executor = garf_executors.setup_executor(
-      source=args.source,
-      fetcher_parameters=context.fetcher_parameters,
-      enable_cache=args.enable_cache,
-      cache_ttl_seconds=args.cache_ttl_seconds,
-    )
-    batch = {query: reader_client.read(query) for query in args.query}
-    query_executor.execute_batch(batch, context, args.parallel_threshold)
   else:
     param_types = ['source', 'macro', 'template']
     outputs = args.output.split(',')
@@ -116,23 +108,24 @@ def main():
       writer_parameters=writer_parameters,
       fetcher_parameters=source_parameters,
     )
-    query_executor = garf_executors.setup_executor(
-      source=args.source,
-      fetcher_parameters=context.fetcher_parameters,
-      enable_cache=args.enable_cache,
-      cache_ttl_seconds=args.cache_ttl_seconds,
-    )
-    if args.parallel_queries and len(args.query) > 1:
-      logger.info('Running queries in parallel')
-      batch = {query: reader_client.read(query) for query in args.query}
-      query_executor.execute_batch(batch, context, args.parallel_threshold)
-    else:
-      if len(args.query) > 1:
-        logger.info('Running queries sequentially')
-      for query in args.query:
-        query_executor.execute(
-          query=reader_client.read(query), title=query, context=context
-        )
+  query_executor = garf_executors.setup_executor(
+    source=args.source,
+    fetcher_parameters=context.fetcher_parameters,
+    enable_cache=args.enable_cache,
+    cache_ttl_seconds=args.cache_ttl_seconds,
+  )
+  batch = {query: reader_client.read(query) for query in args.query}
+  if args.parallel_queries and len(args.query) > 1:
+    logger.info('Running queries in parallel')
+    batch = {query: reader_client.read(query) for query in args.query}
+    query_executor.execute_batch(batch, context, args.parallel_threshold)
+  else:
+    if len(args.query) > 1:
+      logger.info('Running queries sequentially')
+    for query in args.query:
+      query_executor.execute(
+        query=reader_client.read(query), title=query, context=context
+      )
   logging.shutdown()
 
 

{garf_executors-0.1.7 → garf_executors-0.2.0}/garf_executors/executor.py

@@ -15,7 +15,10 @@
 """Defines common functionality between executors."""
 
 import asyncio
+import inspect
+from typing import Optional
 
+from garf_core import report_fetcher
 from opentelemetry import trace
 
 from garf_executors import execution_context
@@ -25,6 +28,14 @@ from garf_executors.telemetry import tracer
 class Executor:
   """Defines common functionality between executors."""
 
+  def __init__(
+    self,
+    preprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
+    postprocessors: Optional[dict[str, report_fetcher.Processor]] = None,
+  ) -> None:
+    self.preprocessors = preprocessors or {}
+    self.postprocessors = postprocessors or {}
+
   @tracer.start_as_current_span('api.execute_batch')
   def execute_batch(
     self,
@@ -34,6 +45,9 @@ class Executor:
   ) -> list[str]:
     """Executes batch of queries for a common context.
 
+    If an executor has any pre/post processors, executes them first while
+    modifying the context.
+
     Args:
       batch: Mapping between query_title and its text.
       context: Execution context.
@@ -44,11 +58,19 @@ class Executor:
     """
     span = trace.get_current_span()
     span.set_attribute('api.parallel_threshold', parallel_threshold)
-    return asyncio.run(
+    _handle_processors(processors=self.preprocessors, context=context)
+    results = asyncio.run(
       self._run(
         batch=batch, context=context, parallel_threshold=parallel_threshold
       )
     )
+    _handle_processors(processors=self.postprocessors, context=context)
+    return results
+
+  def add_preprocessor(
+    self, preprocessors: dict[str, report_fetcher.Processor]
+  ) -> None:
+    self.preprocessors.update(preprocessors)
 
   async def aexecute(
     self,
@@ -85,3 +107,18 @@ class Executor:
       for title, query in batch.items()
     ]
     return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))
+
+
+def _handle_processors(
+  processors: dict[str, report_fetcher.Processor],
+  context: execution_context.ExecutionContext,
+) -> None:
+  for k, processor in processors.items():
+    processor_signature = list(inspect.signature(processor).parameters.keys())
+    if k in context.fetcher_parameters:
+      processor_parameters = {
+        k: v
+        for k, v in context.fetcher_parameters.items()
+        if k in processor_signature
+      }
+      context.fetcher_parameters[k] = processor(**processor_parameters)

{garf_executors-0.1.7 → garf_executors-0.2.0}/garf_executors.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.1.7
+Version: 0.2.0
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0