garf-executors 0.0.3__tar.gz → 0.0.6__tar.gz

{garf_executors-0.0.3 → garf_executors-0.0.6}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.0.3
+Version: 0.0.6
 Summary: Executes queries against API and writes data to local/remote storage.
-Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
+Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.8
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Operating System :: OS Independent
@@ -18,14 +19,18 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: garf-core
 Requires-Dist: garf-io
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
 Provides-Extra: bq
 Requires-Dist: garf-io[bq]; extra == "bq"
 Requires-Dist: pandas; extra == "bq"
 Provides-Extra: sql
 Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
 Requires-Dist: pandas; extra == "sql"
+Provides-Extra: server
+Requires-Dist: fastapi[standard]; extra == "server"
 Provides-Extra: all
-Requires-Dist: garf-executors[bq,sql]; extra == "all"
+Requires-Dist: garf-executors[bq,server,sql]; extra == "all"
 
 # `garf-executors` - One stop-shop for interacting with Reporting APIs.
 

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
   'ApiQueryExecutor',
 ]
 
-__version__ = '0.0.3'
+__version__ = '0.0.6'

garf_executors-0.0.6/garf_executors/api_executor.py

@@ -0,0 +1,132 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for executing Garf queries and writing them to local/remote.
+
+ApiQueryExecutor performs fetching data from API in a form of
+GarfReport and saving it to local/remote storage.
+"""
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+
+from __future__ import annotations
+
+import logging
+
+import pydantic
+
+from garf_core import query_editor, report_fetcher
+from garf_executors import exceptions
+from garf_io import writer
+from garf_io.writers import abs_writer
+
+logger = logging.getLogger(__name__)
+
+
+class ApiExecutionContext(pydantic.BaseModel):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    query_parameters: Parameters to dynamically change query text.
+    fetcher_parameters: Parameters to specify fetching setup.
+    writer: Type of writer to use.
+    writer_parameters: Optional parameters to setup writer.
+  """
+
+  query_parameters: query_editor.GarfQueryParameters | None = None
+  fetcher_parameters: dict[str, str] | None = None
+  writer: str = 'console'
+  writer_parameters: dict[str, str] | None = None
+
+  def model_post_init(self, __context__) -> None:
+    if self.fetcher_parameters is None:
+      self.fetcher_parameters = {}
+    if self.writer_parameters is None:
+      self.writer_parameters = {}
+
+  @property
+  def writer_client(self) -> abs_writer.AbsWriter:
+    writer_client = writer.create_writer(self.writer, **self.writer_parameters)
+    if self.writer == 'bq':
+      _ = writer_client.create_or_get_dataset()
+    if self.writer == 'sheet':
+      writer_client.init_client()
+    return writer_client
+
+
+class ApiQueryExecutor:
+  """Gets data from API and writes them to local/remote storage.
+
+  Attributes:
+      api_client: a client used for connecting to API.
+  """
+
+  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
+    """Initializes ApiQueryExecutor.
+
+    Args:
+        fetcher: Instantiated report fetcher.
+    """
+    self.fetcher = fetcher
+
+  async def aexecute(
+    self, query: str, context: ApiExecutionContext, **kwargs: str
+  ) -> None:
+    """Reads query, extract results and stores them in a specified location.
+
+    Args:
+      query: Location of the query.
+      context: Query execution context.
+    """
+    self.execute(query, context, **kwargs)
+
+  def execute(
+    self,
+    query: str,
+    title: str,
+    context: ApiExecutionContext,
+  ) -> None:
+    """Reads query, extract results and stores them in a specified location.
+
+    Args:
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Raises:
+      GarfExecutorError: When failed to execute query.
+    """
+    try:
+      logger.debug('starting query %s', query)
+      results = self.fetcher.fetch(
+        query_specification=query,
+        args=context.query_parameters,
+        **context.fetcher_parameters,
+      )
+      writer_client = context.writer_client
+      logger.debug(
+        'Start writing data for query %s via %s writer',
+        title,
+        type(writer_client),
+      )
+      writer_client.write(results, title)
+      logger.debug(
+        'Finish writing data for query %s via %s writer',
+        title,
+        type(writer_client),
+      )
+      logger.info('%s executed successfully', title)
+    except Exception as e:
+      logger.error('%s generated an exception: %s', title, str(e))
+      raise exceptions.GarfExecutorError(
+        '%s generated an exception: %s', title, str(e)
+      ) from e

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors/entrypoints/cli.py

@@ -20,14 +20,13 @@ storage.
 from __future__ import annotations
 
 import argparse
-import functools
 import sys
 from concurrent import futures
 
 import garf_executors
 from garf_executors import exceptions
 from garf_executors.entrypoints import utils
-from garf_io import reader, writer
+from garf_io import reader
 
 
 def main():
@@ -35,7 +34,7 @@ def main():
   parser.add_argument('query', nargs='*')
   parser.add_argument('-c', '--config', dest='garf_config', default=None)
   parser.add_argument('--source', dest='source', default=None)
-  parser.add_argument('--output', dest='output', default=None)
+  parser.add_argument('--output', dest='output', default='console')
   parser.add_argument('--input', dest='input', default='file')
   parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
   parser.add_argument('--logger', dest='logger', default='local')
@@ -72,7 +71,6 @@ def main():
     raise exceptions.GarfExecutorError(
       'Please provide one or more queries to run'
     )
-
   config = utils.ConfigBuilder('garf').build(vars(args), kwargs)
   logger.debug('config: %s', config)
 
@@ -81,17 +79,18 @@ def main():
   logger.debug('initialized config: %s', config)
 
   extra_parameters = utils.ParamsParser(['source']).parse(kwargs)
-  query_executor = garf_executors.api_executor.ApiQueryExecutor(
-    concrete_api_fetcher()
-  )
+  source_parameters = extra_parameters.get('source', {})
   reader_client = reader.create_reader(args.input)
 
-  writer_client = writer.create_writer(config.output, **config.writer_params)
-  if config.output == 'bq':
-    _ = writer_client.create_or_get_dataset()
-  if config.output == 'sheet':
-    writer_client.init_client()
-
+  context = garf_executors.api_executor.ApiExecutionContext(
+    query_parameters=config.params,
+    writer=args.output,
+    writer_parameters=config.writer_params,
+    fetcher_parameters=source_parameters,
+  )
+  query_executor = garf_executors.api_executor.ApiQueryExecutor(
+    concrete_api_fetcher(**source_parameters)
+  )
   if args.parallel_queries:
     logger.info('Running queries in parallel')
     with futures.ThreadPoolExecutor(args.parallel_threshold) as executor:
@@ -100,27 +99,16 @@ def main():
           query_executor.execute,
           reader_client.read(query),
           query,
-          writer_client,
-          config.params,
-          **extra_parameters.get('source', {}),
+          context,
         ): query
         for query in args.query
       }
       for future in futures.as_completed(future_to_query):
-        query = future_to_query[future]
-        utils.garf_runner(query, future.result, logger)
+        future.result()
   else:
     logger.info('Running queries sequentially')
     for query in args.query:
-      callback = functools.partial(
-        query_executor.execute,
-        reader_client.read(query),
-        query,
-        writer_client,
-        config.params,
-        **extra_parameters.get('source', {}),
-      )
-      utils.garf_runner(query, callback, logger)
+      query_executor.execute(reader_client.read(query), query, context)
 
 
 if __name__ == '__main__':

garf_executors-0.0.6/garf_executors/entrypoints/server.py

@@ -0,0 +1,65 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastAPI endpoint for executing queries."""
+
+import fastapi
+import pydantic
+import uvicorn
+
+import garf_executors
+from garf_executors import exceptions
+
+
+class ApiExecutorRequest(pydantic.BaseModel):
+  """Request for executing a query.
+
+  Attributes:
+    source: Type of API to interact with.
+    query: Query to execute.
+    title: Name of the query used as an output for writing.
+    context: Execution context.
+  """
+
+  source: str
+  query: str
+  title: str
+  context: garf_executors.api_executor.ApiExecutionContext
+
+
+router = fastapi.APIRouter(prefix='/api')
+
+
+@router.post('/execute')
+async def execute(request: ApiExecutorRequest) -> dict[str, str]:
+  if not (concrete_api_fetcher := garf_executors.FETCHERS.get(request.source)):
+    raise exceptions.GarfExecutorError(
+      f'Source {request.source} is not available.'
+    )
+
+  query_executor = garf_executors.api_executor.ApiQueryExecutor(
+    concrete_api_fetcher(**request.context.fetcher_parameters)
+  )
+
+  query_executor.execute(request.query, request.title, request.context)
+
+  return fastapi.responses.JSONResponse(
+    content=fastapi.encoders.jsonable_encoder({'result': 'success'})
+  )
+
+
+if __name__ == '__main__':
+  app = fastapi.FastAPI()
+  app.include_router(router)
+  uvicorn.run(app)

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors/entrypoints/utils.py

@@ -20,9 +20,8 @@ import datetime
 import logging
 import os
 import sys
-import traceback
 from collections.abc import MutableSequence, Sequence
-from typing import Any, Callable, TypedDict
+from typing import Any, TypedDict
 
 import smart_open
 import yaml
@@ -298,21 +297,36 @@ class ParamsParser:
     key = param[0]
     if not identifier or identifier not in key:
       return None
-    provided_identifier, key = key.split('.')
+    provided_identifier, *keys = key.split('.')
+    if len(keys) > 1:
+      raise GarfParamsException(
+        f'{key} is invalid format,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
     provided_identifier = provided_identifier.replace('--', '')
     if provided_identifier not in self.identifiers:
       raise GarfParamsException(
         f'CLI argument {provided_identifier} is not supported'
-        f", supported arguments {', '.join(self.identifiers)}"
+        f', supported arguments {", ".join(self.identifiers)}'
       )
     if provided_identifier != identifier:
       return None
-    key = key.replace('-', '_')
+    key = keys[0].replace('-', '_')
+    if not key:
+      raise GarfParamsException(
+        f'{identifier} {key} is invalid,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
     if len(param) == 2:
       return {key: param[1]}
+    if len(param) == 1:
+      return {key: True}
     raise GarfParamsException(
       f'{identifier} {key} is invalid,'
-      f'--{identifier}.key=value is the correct format'
+      f'`--{identifier}.key=value` or `--{identifier}.key` '
+      'are the correct formats'
     )
 
 
@@ -429,25 +443,6 @@ def _remove_empty_values(dict_object: dict[str, Any]) -> dict[str, Any]:
     return dict_object
 
 
-def garf_runner(query: str, callback: Callable, logger) -> None:
-  try:
-    logger.debug('starting query %s', query)
-    callback()
-    logger.info('%s executed successfully', query)
-  except Exception as e:
-    traceback.print_tb(e.__traceback__)
-    logger.error('%s generated an exception: %s', query, str(e))
-
-
-def postprocessor_runner(query: str, callback: Callable, logger) -> None:
-  try:
-    logger.debug('starting query %s', query)
-    callback()
-    logger.info('%s executed successfully', query)
-  except Exception as e:
-    logger.error('%s generated an exception: %s', query, str(e))
-
-
 def init_logging(
   loglevel: str = 'INFO', logger_type: str = 'local', name: str = __name__
 ) -> logging.Logger:

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors/sql_executor.py

@@ -47,6 +47,13 @@ class SqlAlchemyQueryExecutor(query_editor.TemplateProcessorMixin):
     """
     self.engine = engine
 
+  @classmethod
+  def from_connection_string(
+    cls, connection_string: str
+  ) -> SqlAlchemyQueryExecutor:
+    engine = sqlalchemy.create_engine(connection_string)
+    return cls(engine)
+
   def execute(
     self,
     script_name: str | None,

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: garf-executors
-Version: 0.0.3
+Version: 0.0.6
 Summary: Executes queries against API and writes data to local/remote storage.
-Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
+Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.8
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Operating System :: OS Independent
@@ -18,14 +19,18 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: garf-core
 Requires-Dist: garf-io
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
 Provides-Extra: bq
 Requires-Dist: garf-io[bq]; extra == "bq"
 Requires-Dist: pandas; extra == "bq"
 Provides-Extra: sql
 Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
 Requires-Dist: pandas; extra == "sql"
+Provides-Extra: server
+Requires-Dist: fastapi[standard]; extra == "server"
 Provides-Extra: all
-Requires-Dist: garf-executors[bq,sql]; extra == "all"
+Requires-Dist: garf-executors[bq,server,sql]; extra == "all"
 
 # `garf-executors` - One stop-shop for interacting with Reporting APIs.
 

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors.egg-info/SOURCES.txt

@@ -14,4 +14,5 @@ garf_executors.egg-info/requires.txt
 garf_executors.egg-info/top_level.txt
 garf_executors/entrypoints/__init__.py
 garf_executors/entrypoints/cli.py
+garf_executors/entrypoints/server.py
 garf_executors/entrypoints/utils.py

{garf_executors-0.0.3 → garf_executors-0.0.6}/garf_executors.egg-info/requires.txt

@@ -1,13 +1,18 @@
 garf-core
 garf-io
+pyyaml
+pydantic
 
 [all]
-garf-executors[bq,sql]
+garf-executors[bq,server,sql]
 
 [bq]
 garf-io[bq]
 pandas
 
+[server]
+fastapi[standard]
+
 [sql]
 garf-io[sqlalchemy]
 pandas

{garf_executors-0.0.3 → garf_executors-0.0.6}/pyproject.toml

@@ -7,9 +7,12 @@ name = "garf-executors"
 dependencies = [
   "garf-core",
   "garf-io",
+  "pyyaml",
+  "pydantic",
 ]
 authors = [
   {name = "Google Inc. (gTech gPS CSE team)", email = "no-reply@google.com"},
+  {name = "Andrei Markin", email = "andrey.markin.ppc@gmail.com"},
 ]
 requires-python = ">=3.8"
 description = "Executes queries against API and writes data to local/remote storage."
@@ -22,6 +25,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Intended Audience :: Developers",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Operating System :: OS Independent",
@@ -41,8 +45,11 @@ sql=[
   "garf-io[sqlalchemy]",
   "pandas",
 ]
+server=[
+  "fastapi[standard]",
+]
 all = [
-  "garf-executors[bq,sql]"
+  "garf-executors[bq,sql,server]"
 ]
 [project.scripts]
 garf="garf_executors.entrypoints.cli:main"

garf_executors-0.0.3/garf_executors/api_executor.py

@@ -1,98 +0,0 @@
-# Copyright 2024 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for executing Garf queries and writing them to local/remote.
-
-ApiQueryExecutor performs fetching data from API in a form of
-GarfReport and saving it to local/remote storage.
-"""
-# pylint: disable=C0330, g-bad-import-order, g-multiple-import
-
-from __future__ import annotations
-
-import logging
-
-from garf_core import report_fetcher
-from garf_io.writers import abs_writer, console_writer
-
-logger = logging.getLogger(__name__)
-
-
-class ApiQueryExecutor:
-  """Gets data from API and writes them to local/remote storage.
-
-  Attributes:
-      api_client: a client used for connecting to API.
-  """
-
-  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
-    """Initializes QueryExecutor.
-
-    Args:
-        fetcher: Instantiated report fetcher.
-    """
-    self.fetcher = fetcher
-
-  async def aexecute(
-    self,
-    query_text: str,
-    query_name: str,
-    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
-    args: dict[str, str] | None = None,
-    **kwargs: str,
-  ) -> None:
-    """Reads query, extract results and stores them in a specified location.
-
-    Args:
-        query_text: Text for the query.
-        query_name: Identifier of a query.
-        customer_ids: All accounts for which query will be executed.
-        writer_client: Client responsible for writing data to local/remote
-            location.
-        args: Arguments that need to be passed to the query.
-        optimize_performance: strategy for speeding up query execution
-            ("NONE", "PROTOBUF", "BATCH", "BATCH_PROTOBUF").
-    """
-    self.execute(query_text, query_name, writer_client, args, **kwargs)
-
-  def execute(
-    self,
-    query_text: str,
-    query_name: str,
-    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
-    args: dict[str, str] | None = None,
-    **kwargs: str,
-  ) -> None:
-    """Reads query, extract results and stores them in a specified location.
-
-    Args:
-        query_text: Text for the query.
-        query_name: Identifier of a query.
-        writer_client: Client responsible for writing data to local/remote
-            location.
-        args: Arguments that need to be passed to the query.
-    """
-    results = self.fetcher.fetch(
-      query_specification=query_text, args=args, **kwargs
-    )
-    logger.debug(
-      'Start writing data for query %s via %s writer',
-      query_name,
-      type(writer_client),
-    )
-    writer_client.write(results, query_name)
-    logger.debug(
-      'Finish writing data for query %s via %s writer',
-      query_name,
-      type(writer_client),
-    )