garf-executors 0.1.0__tar.gz → 0.1.3__tar.gz

{garf_executors-0.1.0 → garf_executors-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.1.0
+Version: 0.1.3
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/__init__.py

@@ -24,7 +24,10 @@ from garf_executors.telemetry import tracer
 
 @tracer.start_as_current_span('setup_executor')
 def setup_executor(
-  source: str, fetcher_parameters: dict[str, str]
+  source: str,
+  fetcher_parameters: dict[str, str | int | bool],
+  enable_cache: bool = False,
+  cache_ttl_seconds: int = 3600,
 ) -> type[executor.Executor]:
   """Initializes executors based on a source and parameters."""
   if source == 'bq':
@@ -40,7 +43,11 @@ def setup_executor(
   else:
     concrete_api_fetcher = fetchers.get_report_fetcher(source)
     query_executor = ApiQueryExecutor(
-      concrete_api_fetcher(**fetcher_parameters)
+      concrete_api_fetcher(
+        **fetcher_parameters,
+        enable_cache=enable_cache,
+        cache_ttl_seconds=cache_ttl_seconds,
+      )
     )
   return query_executor
 
@@ -50,4 +57,4 @@ __all__ = [
   'ApiExecutionContext',
 ]
 
-__version__ = '0.1.0'
+__version__ = '0.1.3'

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/api_executor.py

@@ -20,9 +20,11 @@ GarfReport and saving it to local/remote storage.
 
 from __future__ import annotations
 
+import asyncio
 import logging
 
 from garf_core import report_fetcher
+from opentelemetry import trace
 
 from garf_executors import exceptions, execution_context, executor, fetchers
 from garf_executors.telemetry import tracer
@@ -76,7 +78,7 @@ class ApiQueryExecutor(executor.Executor):
     Returns:
       Result of writing the report.
     """
-    return await self.execute(query, context, title, context)
+    return await asyncio.to_thread(self.execute, query, title, context)
 
   @tracer.start_as_current_span('api.execute')
   def execute(
@@ -98,7 +100,12 @@ class ApiQueryExecutor(executor.Executor):
     Raises:
       GarfExecutorError: When failed to execute query.
     """
+    span = trace.get_current_span()
+    span.set_attribute('fetcher', self.fetcher.__class__.__name__)
+    span.set_attribute('api_client', self.fetcher.api_client.__class__.__name__)
     try:
+      span.set_attribute('query_title', title)
+      span.set_attribute('query_text', query)
       logger.debug('starting query %s', query)
       results = self.fetcher.fetch(
         query_specification=query,

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/entrypoints/cli.py

@@ -28,8 +28,13 @@ from garf_io import reader
 import garf_executors
 from garf_executors import config, exceptions
 from garf_executors.entrypoints import utils
+from garf_executors.entrypoints.tracer import initialize_tracer
+from garf_executors.telemetry import tracer
 
+initialize_tracer()
 
+
+@tracer.start_as_current_span('garf.entrypoints.cli')
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('query', nargs='*')
@@ -49,9 +54,19 @@ def main():
   parser.add_argument('--dry-run', dest='dry_run', action='store_true')
   parser.add_argument('-v', '--version', dest='version', action='store_true')
   parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
+    '--parallel-threshold', dest='parallel_threshold', default=10, type=int
+  )
+  parser.add_argument(
+    '--enable-cache', dest='enable_cache', action='store_true'
+  )
+  parser.add_argument(
+    '--cache-ttl-seconds',
+    dest='cache_ttl_seconds',
+    default=3600,
+    type=int,
   )
   parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(enable_cache=False)
   parser.set_defaults(dry_run=False)
   args, kwargs = parser.parse_known_args()
 
@@ -74,10 +89,13 @@ def main():
         f'No execution context found for source {args.source} in {config_file}'
       )
     query_executor = garf_executors.setup_executor(
-      args.source, context.fetcher_parameters
+      source=args.source,
+      fetcher_parameters=context.fetcher_parameters,
+      enable_cache=args.enable_cache,
+      cache_ttl_seconds=args.cache_ttl_seconds,
     )
     batch = {query: reader_client.read(query) for query in args.query}
-    query_executor.execute_batch(batch, context, args.parallel_queries)
+    query_executor.execute_batch(batch, context, args.parallel_threshold)
   else:
     extra_parameters = utils.ParamsParser(
       ['source', args.output, 'macro', 'template']
@@ -94,12 +112,15 @@ def main():
       fetcher_parameters=source_parameters,
     )
     query_executor = garf_executors.setup_executor(
-      args.source, context.fetcher_parameters
+      source=args.source,
+      fetcher_parameters=context.fetcher_parameters,
+      enable_cache=args.enable_cache,
+      cache_ttl_seconds=args.cache_ttl_seconds,
     )
     if args.parallel_queries:
       logger.info('Running queries in parallel')
       batch = {query: reader_client.read(query) for query in args.query}
-      query_executor.execute_batch(batch, context, args.parallel_queries)
+      query_executor.execute_batch(batch, context, args.parallel_threshold)
     else:
       logger.info('Running queries sequentially')
       for query in args.query:

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/entrypoints/server.py

@@ -94,7 +94,7 @@ async def execute(request: ApiExecutorRequest) -> ApiExecutorResponse:
 
 
 @app.post('/api/execute:batch')
-async def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
+def execute_batch(request: ApiExecutorRequest) -> ApiExecutorResponse:
   query_executor = garf_executors.setup_executor(
     request.source, request.context.fetcher_parameters
   )

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/entrypoints/tracer.py

@@ -34,11 +34,9 @@ def initialize_tracer():
 
   tracer_provider = TracerProvider(resource=resource)
 
-  otlp_processor = BatchSpanProcessor(
-    OTLPSpanExporter(
-      endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'), insecure=True
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    otlp_processor = BatchSpanProcessor(
+      OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
     )
-  )
-  tracer_provider.add_span_processor(otlp_processor)
-
+    tracer_provider.add_span_processor(otlp_processor)
   trace.set_tracer_provider(tracer_provider)

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/execution_context.py

@@ -42,8 +42,8 @@ class ExecutionContext(pydantic.BaseModel):
   query_parameters: query_editor.GarfQueryParameters | None = pydantic.Field(
     default_factory=dict
   )
-  fetcher_parameters: dict[str, str | list[str | int]] | None = pydantic.Field(
-    default_factory=dict
+  fetcher_parameters: dict[str, str | bool | int | list[str | int]] | None = (
+    pydantic.Field(default_factory=dict)
   )
   writer: str | None = None
   writer_parameters: dict[str, str] | None = pydantic.Field(

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors/executor.py

@@ -14,14 +14,18 @@
 
 """Defines common functionality between executors."""
 
-from concurrent import futures
+import asyncio
+
+from opentelemetry import trace
 
 from garf_executors import execution_context
+from garf_executors.telemetry import tracer
 
 
 class Executor:
   """Defines common functionality between executors."""
 
+  @tracer.start_as_current_span('api.execute_batch')
   def execute_batch(
     self,
     batch: dict[str, str],
@@ -38,17 +42,28 @@ class Executor:
     Returns:
       Results of execution.
     """
-    results = []
-    with futures.ThreadPoolExecutor(max_workers=parallel_threshold) as executor:
-      future_to_query = {
-        executor.submit(
-          self.execute,
-          query,
-          title,
-          context,
-        ): query
-        for title, query in batch.items()
-      }
-      for future in futures.as_completed(future_to_query):
-        results.append(future.result())
-    return results
+    span = trace.get_current_span()
+    span.set_attribute('api.parallel_threshold', parallel_threshold)
+    return asyncio.run(
+      self._run(
+        batch=batch, context=context, parallel_threshold=parallel_threshold
+      )
+    )
+
+  async def _run(
+    self,
+    batch: dict[str, str],
+    context: execution_context.ExecutionContext,
+    parallel_threshold: int,
+  ):
+    semaphore = asyncio.Semaphore(value=parallel_threshold)
+
+    async def run_with_semaphore(fn):
+      async with semaphore:
+        return await fn
+
+    tasks = [
+      self.aexecute(query=query, title=title, context=context)
+      for title, query in batch.items()
+    ]
+    return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))

{garf_executors-0.1.0 → garf_executors-0.1.3}/garf_executors.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.1.0
+Version: 0.1.3
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0