garf-executors 1.0.2__py3-none-any.whl → 1.2.0__py3-none-any.whl

garf/executors/{workflow.py → workflows/workflow.py}

@@ -11,10 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Workflow specifies steps of end-to-end fetching and processing."""
+
 from __future__ import annotations
 
+import copy
 import os
 import pathlib
+import re
+from collections import defaultdict
+from typing import Any
 
 import pydantic
 import smart_open
@@ -37,6 +43,13 @@ class QueryPath(pydantic.BaseModel):
   """Path file with query."""
 
   path: str
+  prefix: str | None = None
+
+  @property
+  def full_path(self) -> str:
+    if self.prefix:
+      return re.sub('/$', '', self.prefix) + '/' + self.path
+    return self.path
 
 
 class QueryDefinition(pydantic.BaseModel):
@@ -65,11 +78,13 @@ class ExecutionStep(ExecutionContext):
     alias: Optional alias to identify execution step.
     queries: Queries to run for a particular fetcher.
     context: Execution context for queries and fetcher.
+    parallel_threshold: Max allowed parallelism for the queries in the step.
   """
 
   fetcher: str | None = None
   alias: str | None = pydantic.Field(default=None, pattern=r'^[a-zA-Z0-9_]+$')
   queries: list[QueryPath | QueryDefinition | QueryFolder] | None = None
+  parallel_threshold: int | None = None
 
   @property
   def context(self) -> ExecutionContext:
@@ -86,17 +101,43 @@ class Workflow(pydantic.BaseModel):
 
   Attributes:
     steps: Contains one or several fetcher executions.
+    context: Query and fetcher parameters to overwrite in steps.
   """
 
   steps: list[ExecutionStep]
+  context: ExecutionContext | None = None
+
+  def model_post_init(self, __context__) -> None:
+    if context := self.context:
+      custom_parameters = defaultdict(dict)
+      if custom_macros := context.query_parameters.macro:
+        custom_parameters['query_parameters']['macro'] = custom_macros
+      if custom_templates := context.query_parameters.template:
+        custom_parameters['query_parameters']['template'] = custom_templates
+      if custom_fetcher_parameters := context.fetcher_parameters:
+        custom_parameters['fetcher_parameters'] = custom_fetcher_parameters
+      if custom_writer_parameters := context.writer_parameters:
+        custom_parameters['writer_parameters'] = custom_writer_parameters
+
+      if custom_parameters:
+        steps = self.steps
+        for i, step in enumerate(steps):
+          res = _merge_dicts(
+            step.model_dump(exclude_none=True), dict(custom_parameters)
+          )
+          steps[i] = ExecutionStep(**res)
 
   @classmethod
-  def from_file(cls, path: str | pathlib.Path | os.PathLike[str]) -> Workflow:
+  def from_file(
+    cls,
+    path: str | pathlib.Path | os.PathLike[str],
+    context: ExecutionContext | None = None,
+  ) -> Workflow:
     """Builds workflow from local or remote yaml file."""
     with smart_open.open(path, 'r', encoding='utf-8') as f:
       data = yaml.safe_load(f)
     try:
-      return Workflow(**data)
+      return Workflow(steps=data.get('steps'), context=context)
     except pydantic.ValidationError as e:
       raise GarfWorkflowError(f'Incorrect workflow:\n {e}') from e
 
@@ -104,6 +145,22 @@ class Workflow(pydantic.BaseModel):
     """Saves workflow to local or remote yaml file."""
     with smart_open.open(path, 'w', encoding='utf-8') as f:
       yaml.dump(
-        self.model_dump(exclude_none=True).get('steps'), f, encoding='utf-8'
+        self.model_dump(exclude_none=True), f, encoding='utf-8', sort_keys=False
       )
     return f'Workflow is saved to {str(path)}'
+
+
+def _merge_dicts(
+  dict1: dict[str, Any], dict2: dict[str, Any]
+) -> dict[str, Any]:
+  result = copy.deepcopy(dict1)
+  for key, value in dict2.items():
+    if (
+      key in result
+      and isinstance(result[key], dict)
+      and isinstance(value, dict)
+    ):
+      result[key] = _merge_dicts(result[key], value)
+    else:
+      result[key] = value
+  return result

garf/executors/workflows/workflow_runner.py

@@ -0,0 +1,176 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runs garf workflow."""
+
+from __future__ import annotations
+
+import logging
+import pathlib
+import re
+from typing import Final
+
+import yaml
+from garf.executors import exceptions, setup
+from garf.executors.telemetry import tracer
+from garf.executors.workflows import workflow
+from garf.io import reader
+
+logger = logging.getLogger(__name__)
+
+_REMOTE_FILES_PATTERN: Final[str] = (
+  '^(http|gs|s3|aruze|hdfs|webhdfs|ssh|scp|sftp)'
+)
+_SCRIPT_PATH = pathlib.Path(__file__).parent
+
+
+class WorkflowRunner:
+  """Runs garf workflow.
+
+  Attributes:
+    workflow: Workflow to execute.
+    wf_parent: Optional location of a workflow file.
+    parallel_threshold: Max allowed parallelism for the queries in the workflow.
+  """
+
+  def __init__(
+    self,
+    execution_workflow: workflow.Workflow,
+    wf_parent: pathlib.Path | str,
+    parallel_threshold: int = 10,
+  ) -> None:
+    """Initializes WorkflowRunner."""
+    self.workflow = execution_workflow
+    self.wf_parent = wf_parent
+    self.parallel_threshold = parallel_threshold
+
+  @classmethod
+  def from_file(
+    cls,
+    workflow_file: str | pathlib.Path,
+  ) -> WorkflowRunner:
+    """Initialized Workflow runner from a local or remote file."""
+    if isinstance(workflow_file, str):
+      workflow_file = pathlib.Path(workflow_file)
+    execution_workflow = workflow.Workflow.from_file(workflow_file)
+    return cls(
+      execution_workflow=execution_workflow, wf_parent=workflow_file.parent
+    )
+
+  def run(
+    self,
+    enable_cache: bool = False,
+    cache_ttl_seconds: int = 3600,
+    selected_aliases: list[str] | None = None,
+    skipped_aliases: list[str] | None = None,
+    simulate: bool = False,
+  ) -> list[str]:
+    skipped_aliases = skipped_aliases or []
+    selected_aliases = selected_aliases or []
+    reader_client = reader.create_reader('file')
+    execution_results = []
+    logger.info('Starting Garf Workflow...')
+    for i, step in enumerate(self.workflow.steps, 1):
+      step_name = f'{i}-{step.fetcher}'
+      if step.alias:
+        step_name = f'{step_name}-{step.alias}'
+      if step.alias in skipped_aliases:
+        logger.warning(
+          'Skipping step %d, fetcher: %s, alias: %s',
+          i,
+          step.fetcher,
+          step.alias,
+        )
+        continue
+      if selected_aliases and step.alias not in selected_aliases:
+        logger.warning(
+          'Skipping step %d, fetcher: %s, alias: %s',
+          i,
+          step.fetcher,
+          step.alias,
+        )
+        continue
+      with tracer.start_as_current_span(step_name):
+        logger.info(
+          'Running step %d, fetcher: %s, alias: %s', i, step.fetcher, step.alias
+        )
+        query_executor = setup.setup_executor(
+          source=step.fetcher,
+          fetcher_parameters=step.fetcher_parameters,
+          enable_cache=enable_cache,
+          cache_ttl_seconds=cache_ttl_seconds,
+          simulate=simulate,
+          writers=step.writer,
+          writer_parameters=step.writer_parameters,
+        )
+        batch = {}
+        if not (queries := step.queries):
+          logger.error('Please provide one or more queries to run')
+          raise exceptions.GarfExecutorError(
+            'Please provide one or more queries to run'
+          )
+        for query in queries:
+          if isinstance(query, workflow.QueryPath):
+            query_path = query.full_path
+            if re.match(_REMOTE_FILES_PATTERN, query_path):
+              batch[query.path] = reader_client.read(query_path)
+            else:
+              if not query.prefix:
+                query_path = self.wf_parent / pathlib.Path(query.path)
+              if not query_path.exists():
+                raise workflow.GarfWorkflowError(
+                  f'Query: {query_path} not found'
+                )
+              batch[query.path] = reader_client.read(query_path)
+          elif isinstance(query, workflow.QueryFolder):
+            query_path = self.wf_parent / pathlib.Path(query.folder)
+            if not query_path.exists():
+              raise workflow.GarfWorkflowError(
+                f'Folder: {query_path} not found'
+              )
+            for p in query_path.rglob('*'):
+              if p.suffix == '.sql':
+                batch[p.stem] = reader_client.read(p)
+          else:
+            batch[query.query.title] = query.query.text
+        query_executor.execute_batch(
+          batch,
+          step.context,
+          step.parallel_threshold or self.parallel_threshold,
+        )
+        execution_results.append(step_name)
+    return execution_results
+
+  def compile(self, path: str | pathlib.Path) -> str:
+    """Saves workflow with expanded anchors."""
+    return self.workflow.save(path)
+
+  def deploy(self, path: str | pathlib.Path) -> str:
+    """Prepares workflow for deployment to Google Cloud Workflows."""
+    wf = self.workflow.model_dump(exclude_none=True).get('steps')
+    with open(_SCRIPT_PATH / 'gcp_workflow.yaml', 'r', encoding='utf-8') as f:
+      cloud_workflow_run_template = yaml.safe_load(f)
+    init = {
+      'init': {
+        'assign': [{'pairs': wf}],
+      },
+    }
+    cloud_workflow = {
+      'main': {
+        'params': [],
+        'steps': [init, cloud_workflow_run_template],
+      },
+    }
+    with open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(cloud_workflow, f, sort_keys=False)
+    return f'Workflow is saved to {path}'

{garf_executors-1.0.2.dist-info → garf_executors-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-executors
-Version: 1.0.2
+Version: 1.2.0
 Summary: Executes queries against API and writes data to local/remote storage.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0
@@ -36,8 +36,15 @@ Provides-Extra: gcp
 Requires-Dist: opentelemetry-exporter-gcp-trace; extra == "gcp"
 Provides-Extra: server
 Requires-Dist: fastapi[standard]; extra == "server"
+Requires-Dist: pydantic-settings; extra == "server"
 Requires-Dist: opentelemetry-instrumentation-fastapi; extra == "server"
 Requires-Dist: typer; extra == "server"
+Requires-Dist: grpcio-reflection; extra == "server"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-mock; extra == "tests"
+Requires-Dist: pytest-xdist; extra == "tests"
+Requires-Dist: pytest-grpc; extra == "tests"
 Provides-Extra: all
 Requires-Dist: garf-executors[bq,gcp,server,sql]; extra == "all"
 

{garf_executors-1.0.2.dist-info → garf_executors-1.2.0.dist-info}/RECORD

@@ -1,23 +1,27 @@
-garf/executors/__init__.py,sha256=dO6T9Z9Q5zc3BszqaTQE8fK5weyV_RTmj8ElV0mUFhQ,1941
-garf/executors/api_executor.py,sha256=bLhG9FLgAfdO16BW22YhAq0SXY8c-WuNW13e6bUu_VE,4380
-garf/executors/bq_executor.py,sha256=f5jQllFAE3b_ajQlosZfecovnrIDfyOAO-k8_AjzRqQ,5774
+garf/executors/__init__.py,sha256=i7udlIWHIzMtLJmErjJJGiuauAGlApUlyB2sdX8gR4I,824
+garf/executors/api_executor.py,sha256=96slSspngBQ2_R15gLxaUd565EqCg7syyC2yf2oK6ic,7118
+garf/executors/bq_executor.py,sha256=-LlDFNF-OvPy-9_QzY1d58emOs5eehsgIDPYUkm-H1A,6696
 garf/executors/config.py,sha256=w5g9EYabPtK-6CPl3G87owUvBiqi0_r_aeAwISpK-zw,1720
 garf/executors/exceptions.py,sha256=U_7Q2ZMOUf89gzZd2pw7y3g7i1NeByPPKfpZ3q7p3ZU,662
-garf/executors/execution_context.py,sha256=us_S-x2jsJOBo4Vm78DhCgcgnu_HPc-f9-q_XI_eowU,3840
+garf/executors/execution_context.py,sha256=_T9sEkkImTjtdwJcnxbCEZXA6cpXh5GVs9CCGcGdGcw,3380
 garf/executors/executor.py,sha256=SpPONsYHO49WbZ2HRjTKujrNH64BbrSfKcZSn4Dcd6o,3830
-garf/executors/fetchers.py,sha256=Jtdl9A2G4xEM-pnUg_fWv3cltcMpM7JjdTcLeJK5AVY,2604
-garf/executors/garf_pb2.py,sha256=mYvBYcAnZtyDflXGN2GZLM2KM0Nv9hoJs55zfQU_l1o,2564
-garf/executors/garf_pb2_grpc.py,sha256=w8D_r3wpj1ZZstkIFogY679-lSCcL2iZQ4QLO8IfToY,3359
-garf/executors/query_processor.py,sha256=xCTkm2V0iMfsjJ0ydwyVRFFiZh1ilhJKzglYlReRKAk,2250
-garf/executors/sql_executor.py,sha256=Z1E7aL9HiYcqfUJMuESlu4iZjVy5hNMvplu2hLotK9I,4780
+garf/executors/fetchers.py,sha256=JemMM4FU4-Cpp2SxmMBtLgHgGy2gDsQSuuAozTh7Yjw,4477
+garf/executors/garf_pb2.py,sha256=OIKC7NGErbUckhB4pQ6HycB9My5X3FvaImARnvhPExM,3450
+garf/executors/garf_pb2_grpc.py,sha256=repGTh-ZDnNyAxMcJxAf0cLfr_JjX2AzZkY6PfZy0xM,4957
+garf/executors/query_processor.py,sha256=IM5qXBNYJSUlsY2djYx8WDeX367cMhI1ZrITT22TcvI,2932
+garf/executors/setup.py,sha256=txQtkpMNQ5WlYF2MzJdvPop07F1IqNN_X97mzedghDc,2604
+garf/executors/sql_executor.py,sha256=TzDzVxpNjyniu9ZHubcLUfYg0igGc1cOUXH9ETt1WL8,5226
 garf/executors/telemetry.py,sha256=wLWAdJZmGinffIMv5FZNKaAUusgACTvokwhMFz2UCQ0,747
-garf/executors/workflow.py,sha256=WFsTV916mw_DsWttrLc8tNnDl0vwJAQwVlIHRdNaNDw,3007
 garf/executors/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-garf/executors/entrypoints/cli.py,sha256=8vk8jElesqqn0q2Te3v6W1V-kts-28N5sncovaZqq3o,6699
-garf/executors/entrypoints/grpc_server.py,sha256=__y2uR2pwlnYy1YFm-2EjFYxBKZmN1a9XY9MNsfrhag,2295
-garf/executors/entrypoints/server.py,sha256=vXGz-f7C1aweaMNJTJEjZLIg42d0T01HPG_lxlj-FPs,3472
-garf/executors/entrypoints/tracer.py,sha256=-UM1RNtW90jFsQiavVzhIWzB3cOCTcCwc6v7YIVflDk,1912
+garf/executors/entrypoints/cli.py,sha256=x7RAQngeb3rr3rIhJ3d01q38G6DhTQYgt6p3nSPegxg,6123
+garf/executors/entrypoints/grpc_server.py,sha256=O7hinPjlVurSJjzudDFvVjkxX7xJHRcwFk4K4aJK0uo,2783
+garf/executors/entrypoints/server.py,sha256=rLnOgZqeWwZ9UsJZ8LXDvZ101JrUIwMTpDzC5t-6wsY,4988
+garf/executors/entrypoints/tracer.py,sha256=Oug3tePD8Yg5x6r1KNDx8RL_yAML0egy1DEFDobW8Uk,2792
 garf/executors/entrypoints/utils.py,sha256=5XiGR2IOxdzAOY0lEWUeUV7tIpKBGRnQaIwBYvzQB7c,4337
+garf/executors/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+garf/executors/workflows/gcp_workflow.yaml,sha256=KKfyFaBihFiLrsNhZct0cccxMV8CnVJUsxBsXQ8VP-g,1743
+garf/executors/workflows/workflow.py,sha256=xqPFHq6xJSWhpbw47U1dUx9AQpYJyMGqA6pRG-4CFQo,4952
+garf/executors/workflows/workflow_runner.py,sha256=A3NxfBq9TgXs9CCaK8MzJBHVLTCDtF1sSuy_BLWbQco,5929
 garf_executors/__init__.py,sha256=5Ol67ktUcC0q5d5pGblYfdlAsraJC-Gcr2U0uCN6rSs,772
 garf_executors/api_executor.py,sha256=lmrPn6aheryM2jLRL2enU8GuKVUod3kEVkMYgjXn5EM,785
 garf_executors/bq_executor.py,sha256=wQ8pd4d6dMByHtYl_i-FbebPKO5WRcvC_y9asG8H3Zk,784
@@ -35,8 +39,8 @@ garf_executors/entrypoints/grcp_server.py,sha256=HiYsfk31OgkPA4jcED3htJGidkRZH5N
 garf_executors/entrypoints/server.py,sha256=JZCklhqx74PIhc4GIoOr2nNZz9n7QCfIm_vGyd3Q3dQ,815
 garf_executors/entrypoints/tracer.py,sha256=JlDSgeDP0Q5Lk_pZLASDXPgZCPaKkWqZgaWOQ7JB-Bs,815
 garf_executors/entrypoints/utils.py,sha256=iF-LBfKjrAhEW6HShh69RCPWVPC4Rf8iV5JEYSMhsx0,814
-garf_executors-1.0.2.dist-info/METADATA,sha256=zNOfNeIWJiEoaWz2ONVf0gQeVRc5q6DM8CxnUxmoB90,3303
-garf_executors-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-garf_executors-1.0.2.dist-info/entry_points.txt,sha256=0IBZun3_hC4HYU-1krlbjTArZym3phu4jxYXs809ilw,61
-garf_executors-1.0.2.dist-info/top_level.txt,sha256=UaHdWdgQhbiHyRzpYC-vW3Q7pdgbxXvTTBvDA655Jq4,20
-garf_executors-1.0.2.dist-info/RECORD,,
+garf_executors-1.2.0.dist-info/METADATA,sha256=s8o4jKBZBT6uAxL6NMoU5pgYGv2JgEHx2f7LVG_jDj0,3605
+garf_executors-1.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+garf_executors-1.2.0.dist-info/entry_points.txt,sha256=0IBZun3_hC4HYU-1krlbjTArZym3phu4jxYXs809ilw,61
+garf_executors-1.2.0.dist-info/top_level.txt,sha256=UaHdWdgQhbiHyRzpYC-vW3Q7pdgbxXvTTBvDA655Jq4,20
+garf_executors-1.2.0.dist-info/RECORD,,

{garf_executors-1.0.2.dist-info → garf_executors-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any