garf-executors 0.2.3__py3-none-any.whl → 1.0.2__py3-none-any.whl

garf/executors/query_processor.py

@@ -0,0 +1,61 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+
+from garf.core import query_editor
+from garf.executors import exceptions, execution_context
+
+
+def process_gquery(
+  context: execution_context.ExecutionContext,
+) -> execution_context.ExecutionContext:
+  for k, v in context.fetcher_parameters.items():
+    if isinstance(v, str) and v.startswith('gquery'):
+      no_writer_context = context.model_copy(update={'writer': None})
+      try:
+        _, alias, query = v.split(':', maxsplit=3)
+      except ValueError:
+        raise exceptions.GarfExecutorError(
+          f'Incorrect gquery format, should be gquery:alias:query, got {v}'
+        )
+      if alias == 'sqldb':
+        from garf.executors import sql_executor
+
+        gquery_executor = sql_executor.SqlAlchemyQueryExecutor(
+          **context.fetcher_parameters
+        )
+      elif alias == 'bq':
+        from garf.executors import bq_executor
+
+        gquery_executor = bq_executor.BigQueryExecutor(
+          **context.fetcher_parameters
+        )
+      else:
+        raise exceptions.GarfExecutorError(
+          f'Unsupported alias for gquery: {alias}'
+        )
+      with contextlib.suppress(query_editor.GarfResourceError):
+        query_spec = query_editor.QuerySpecification(
+          text=query, args=context.query_parameters
+        ).generate()
+        if len(columns := [c for c in query_spec.column_names if c != '_']) > 1:
+          raise exceptions.GarfExecutorError(
+            f'Multiple columns in gquery: {columns}'
+          )
+      res = gquery_executor.execute(
+        query=query, title='gquery', context=no_writer_context
+      )
+      context.fetcher_parameters[k] = res.to_list(row_type='scalar')
+  return context

garf/executors/sql_executor.py

@@ -0,0 +1,142 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Defines mechanism for executing queries via SqlAlchemy."""
+
+from __future__ import annotations
+
+try:
+  import sqlalchemy
+except ImportError as e:
+  raise ImportError(
+    'Please install garf-executors with sqlalchemy support '
+    '- `pip install garf-executors[sqlalchemy]`'
+  ) from e
+
+import logging
+import re
+import uuid
+
+import pandas as pd
+from garf.core import query_editor, report
+from garf.executors import exceptions, execution_context, executor
+from garf.executors.telemetry import tracer
+from opentelemetry import trace
+
+logger = logging.getLogger(__name__)
+
+
+class SqlAlchemyQueryExecutorError(exceptions.GarfExecutorError):
+  """Error when SqlAlchemyQueryExecutor fails to run query."""
+
+
+class SqlAlchemyQueryExecutor(
+  executor.Executor, query_editor.TemplateProcessorMixin
+):
+  """Handles query execution via SqlAlchemy.
+
+  Attributes:
+      engine: Initialized Engine object to operated on a given database.
+  """
+
+  def __init__(
+    self, engine: sqlalchemy.engine.base.Engine | None = None, **kwargs: str
+  ) -> None:
+    """Initializes executor with a given engine.
+
+    Args:
+        engine: Initialized Engine object to operated on a given database.
+    """
+    self.engine = engine or sqlalchemy.create_engine('sqlite://')
+    super().__init__()
+
+  @classmethod
+  def from_connection_string(
+    cls, connection_string: str | None
+  ) -> SqlAlchemyQueryExecutor:
+    """Creates executor from SqlAlchemy connection string.
+
+    https://docs.sqlalchemy.org/en/20/core/engines.html
+    """
+    engine = sqlalchemy.create_engine(connection_string or 'sqlite://')
+    return cls(engine)
+
+  @tracer.start_as_current_span('sql.execute')
+  def execute(
+    self,
+    query: str,
+    title: str,
+    context: execution_context.ExecutionContext = (
+      execution_context.ExecutionContext()
+    ),
+  ) -> report.GarfReport:
+    """Executes query in a given database via SqlAlchemy.
+
+    Args:
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Returns:
+      Report with data if query returns some data otherwise empty Report.
+    """
+    span = trace.get_current_span()
+    logger.info('Executing script: %s', title)
+    query_text = self.replace_params_template(query, context.query_parameters)
+    with self.engine.begin() as conn:
+      if re.findall(r'(create|update) ', query_text.lower()):
+        try:
+          conn.connection.executescript(query_text)
+          results = report.GarfReport()
+        except Exception as e:
+          raise SqlAlchemyQueryExecutorError(
+            f'Failed to execute query {title}: Reason: {e}'
+          ) from e
+      else:
+        temp_table_name = f'temp_{uuid.uuid4().hex}'
+        query_text = f'CREATE TABLE {temp_table_name} AS {query_text}'
+        conn.connection.executescript(query_text)
+        try:
+          results = report.GarfReport.from_pandas(
+            pd.read_sql(f'SELECT * FROM {temp_table_name}', conn)
+          )
+        except Exception as e:
+          raise SqlAlchemyQueryExecutorError(
+            f'Failed to execute query {title}: Reason: {e}'
+          ) from e
+        finally:
+          conn.connection.execute(f'DROP TABLE {temp_table_name}')
+      if context.writer and results:
+        writer_clients = context.writer_clients
+        if not writer_clients:
+          logger.warning('No writers configured, skipping write operation')
+        else:
+          writing_results = []
+          for writer_client in writer_clients:
+            logger.debug(
+              'Start writing data for query %s via %s writer',
+              title,
+              type(writer_client),
+            )
+            writing_result = writer_client.write(results, title)
+            logger.debug(
+              'Finish writing data for query %s via %s writer',
+              title,
+              type(writer_client),
+            )
+            writing_results.append(writing_result)
+          logger.info('%s executed successfully', title)
+          # Return the last writer's result for backward compatibility
+          return writing_results[-1] if writing_results else None
+      span.set_attribute('execute.num_results', len(results))
+      return results

garf/executors/telemetry.py

@@ -0,0 +1,20 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+from opentelemetry import trace
+
+tracer = trace.get_tracer(
+  instrumenting_module_name='garf.executors',
+)

garf/executors/workflow.py

@@ -0,0 +1,109 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pydantic
+import smart_open
+import yaml
+from garf.executors import exceptions
+from garf.executors.execution_context import ExecutionContext
+
+
+class GarfWorkflowError(exceptions.GarfExecutorError):
+  """Workflow specific exception."""
+
+
+class QueryFolder(pydantic.BaseModel):
+  """Path to folder with queries."""
+
+  folder: str
+
+
+class QueryPath(pydantic.BaseModel):
+  """Path file with query."""
+
+  path: str
+
+
+class QueryDefinition(pydantic.BaseModel):
+  """Definition of a query."""
+
+  query: Query
+
+
+class Query(pydantic.BaseModel):
+  """Query elements.
+
+  Attributes:
+    text: Query text.
+    title: Name of the query.
+  """
+
+  text: str
+  title: str
+
+
+class ExecutionStep(ExecutionContext):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    fetcher: Name of a fetcher to get data from API.
+    alias: Optional alias to identify execution step.
+    queries: Queries to run for a particular fetcher.
+    context: Execution context for queries and fetcher.
+  """
+
+  fetcher: str | None = None
+  alias: str | None = pydantic.Field(default=None, pattern=r'^[a-zA-Z0-9_]+$')
+  queries: list[QueryPath | QueryDefinition | QueryFolder] | None = None
+
+  @property
+  def context(self) -> ExecutionContext:
+    return ExecutionContext(
+      writer=self.writer,
+      writer_parameters=self.writer_parameters,
+      query_parameters=self.query_parameters,
+      fetcher_parameters=self.fetcher_parameters,
+    )
+
+
+class Workflow(pydantic.BaseModel):
+  """Orchestrates execution of queries for multiple fetchers.
+
+  Attributes:
+    steps: Contains one or several fetcher executions.
+  """
+
+  steps: list[ExecutionStep]
+
+  @classmethod
+  def from_file(cls, path: str | pathlib.Path | os.PathLike[str]) -> Workflow:
+    """Builds workflow from local or remote yaml file."""
+    with smart_open.open(path, 'r', encoding='utf-8') as f:
+      data = yaml.safe_load(f)
+    try:
+      return Workflow(**data)
+    except pydantic.ValidationError as e:
+      raise GarfWorkflowError(f'Incorrect workflow:\n {e}') from e
+
+  def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
+    """Saves workflow to local or remote yaml file."""
+    with smart_open.open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(
+        self.model_dump(exclude_none=True).get('steps'), f, encoding='utf-8'
+      )
+    return f'Workflow is saved to {str(path)}'

garf_executors/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2025 Google LLC
+# Copyright 2026 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,50 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Executors to fetch data from various APIs."""
 
-from __future__ import annotations
 
-import importlib
+import warnings
 
-from garf_executors import executor, fetchers
-from garf_executors.api_executor import ApiExecutionContext, ApiQueryExecutor
-from garf_executors.telemetry import tracer
+from garf.executors import *
 
-
-@tracer.start_as_current_span('setup_executor')
-def setup_executor(
-  source: str,
-  fetcher_parameters: dict[str, str | int | bool],
-  enable_cache: bool = False,
-  cache_ttl_seconds: int = 3600,
-) -> type[executor.Executor]:
-  """Initializes executors based on a source and parameters."""
-  if source == 'bq':
-    bq_executor = importlib.import_module('garf_executors.bq_executor')
-    query_executor = bq_executor.BigQueryExecutor(**fetcher_parameters)
-  elif source == 'sqldb':
-    sql_executor = importlib.import_module('garf_executors.sql_executor')
-    query_executor = (
-      sql_executor.SqlAlchemyQueryExecutor.from_connection_string(
-        fetcher_parameters.get('connection_string')
-      )
-    )
-  else:
-    concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    query_executor = ApiQueryExecutor(
-      fetcher=concrete_api_fetcher(
-        **fetcher_parameters,
-        enable_cache=enable_cache,
-        cache_ttl_seconds=cache_ttl_seconds,
-      )
-    )
-  return query_executor
-
-
-__all__ = [
-  'ApiQueryExecutor',
-  'ApiExecutionContext',
-]
-
-__version__ = '0.2.3'
+warnings.warn(
+  "The 'garf_executors' namespace is deprecated. "
+  "Please use 'garf.executors' instead.",
+  DeprecationWarning,
+  stacklevel=2,
+)

garf_executors/api_executor.py

@@ -1,4 +1,4 @@
-# Copyright 2024 Google LLC
+# Copyright 2026 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,127 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Module for executing Garf queries and writing them to local/remote.
 
-ApiQueryExecutor performs fetching data from API in a form of
-GarfReport and saving it to local/remote storage.
-"""
-# pylint: disable=C0330, g-bad-import-order, g-multiple-import
 
-from __future__ import annotations
+import warnings
 
-import logging
+from garf.executors.api_executor import *
 
-from garf_core import report_fetcher
-from opentelemetry import trace
-
-from garf_executors import exceptions, execution_context, executor, fetchers
-from garf_executors.telemetry import tracer
-
-logger = logging.getLogger(__name__)
-
-
-class ApiExecutionContext(execution_context.ExecutionContext):
-  """Common context for executing one or more queries."""
-
-  writer: str | list[str] = 'console'
-
-
-class ApiQueryExecutor(executor.Executor):
-  """Gets data from API and writes them to local/remote storage.
-
-  Attributes:
-      api_client: a client used for connecting to API.
-  """
-
-  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
-    """Initializes ApiQueryExecutor.
-
-    Args:
-        fetcher: Instantiated report fetcher.
-    """
-    self.fetcher = fetcher
-    super().__init__(
-      preprocessors=self.fetcher.preprocessors,
-      postprocessors=self.fetcher.postprocessors,
-    )
-
-  @classmethod
-  def from_fetcher_alias(
-    cls,
-    source: str,
-    fetcher_parameters: dict[str, str] | None = None,
-    enable_cache: bool = False,
-    cache_ttl_seconds: int = 3600,
-  ) -> ApiQueryExecutor:
-    if not fetcher_parameters:
-      fetcher_parameters = {}
-    concrete_api_fetcher = fetchers.get_report_fetcher(source)
-    return ApiQueryExecutor(
-      fetcher=concrete_api_fetcher(
-        **fetcher_parameters,
-        enable_cache=enable_cache,
-        cache_ttl_seconds=cache_ttl_seconds,
-      )
-    )
-
-  @tracer.start_as_current_span('api.execute')
-  def execute(
-    self,
-    query: str,
-    title: str,
-    context: ApiExecutionContext,
-  ) -> str:
-    """Reads query, extract results and stores them in a specified location.
-
-    Args:
-      query: Location of the query.
-      title: Name of the query.
-      context: Query execution context.
-
-    Returns:
-      Result of writing the report.
-
-    Raises:
-      GarfExecutorError: When failed to execute query.
-    """
-    span = trace.get_current_span()
-    span.set_attribute('fetcher.class', self.fetcher.__class__.__name__)
-    span.set_attribute(
-      'api.client.class', self.fetcher.api_client.__class__.__name__
-    )
-    try:
-      span.set_attribute('query.title', title)
-      span.set_attribute('query.text', query)
-      logger.debug('starting query %s', query)
-      results = self.fetcher.fetch(
-        query_specification=query,
-        args=context.query_parameters,
-        **context.fetcher_parameters,
-      )
-      writer_clients = context.writer_clients
-      if not writer_clients:
-        logger.warning('No writers configured, skipping write operation')
-        return None
-      writing_results = []
-      for writer_client in writer_clients:
-        logger.debug(
-          'Start writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        result = writer_client.write(results, title)
-        logger.debug(
-          'Finish writing data for query %s via %s writer',
-          title,
-          type(writer_client),
-        )
-        writing_results.append(result)
-      logger.info('%s executed successfully', title)
-      # Return the last writer's result for backward compatibility
-      return writing_results[-1] if writing_results else None
-    except Exception as e:
-      logger.error('%s generated an exception: %s', title, str(e))
-      raise exceptions.GarfExecutorError(
-        '%s generated an exception: %s', title, str(e)
-      ) from e
+warnings.warn(
+  "The 'garf_executors' namespace is deprecated. "
+  "Please use 'garf.executors' instead.",
+  DeprecationWarning,
+  stacklevel=2,
+)