garf-core 0.1.5.post0__py3-none-any.whl → 0.2.1__py3-none-any.whl

garf_core/__init__.py

@@ -26,4 +26,4 @@ __all__ = [
   'ApiReportFetcher',
 ]
 
-__version__ = '0.1.5.post0'
+__version__ = '0.2.1'

garf_core/api_clients.py

@@ -20,23 +20,24 @@ import contextlib
 import csv
 import json
 import os
-import pathlib
 from collections.abc import Sequence
 from typing import Any, Union
 
 import pydantic
 import requests
+import smart_open
 from typing_extensions import TypeAlias, override
 
 from garf_core import exceptions, query_editor
 
 ApiRowElement: TypeAlias = Union[int, float, str, bool, list, dict, None]
+ApiResponseRow: TypeAlias = dict[str, ApiRowElement]
 
 
 class GarfApiResponse(pydantic.BaseModel):
   """Base class for specifying response."""
 
-  results: list[ApiRowElement]
+  results: list[ApiResponseRow]
 
 
 class GarfApiError(exceptions.GarfError):
@@ -123,7 +124,7 @@ class FakeApiClient(BaseClient):
       GarfApiError: When file with data not found.
     """
     try:
-      with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
+      with smart_open.open(file_location, 'r', encoding='utf-8') as f:
         data = json.load(f)
         return FakeApiClient(data)
     except FileNotFoundError as e:
@@ -143,7 +144,7 @@ class FakeApiClient(BaseClient):
       GarfApiError: When file with data not found.
     """
     try:
-      with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
+      with smart_open.open(file_location, 'r', encoding='utf-8') as f:
         reader = csv.DictReader(f)
         data = []
         for row in reader:

garf_core/parsers.py

@@ -16,16 +16,23 @@
 from __future__ import annotations
 
 import abc
+import ast
 import contextlib
 import functools
 import operator
 from collections.abc import Mapping, MutableSequence
 from typing import Any
 
-from typing_extensions import override
-
 from garf_core import api_clients, exceptions, query_editor
 
+VALID_VIRTUAL_COLUMN_OPERATORS = (
+  ast.BinOp,
+  ast.UnaryOp,
+  ast.operator,
+  ast.Constant,
+  ast.Expression,
+)
+
 
 class BaseParser(abc.ABC):
   """An interface for all parsers to implement."""
@@ -48,44 +55,101 @@ class BaseParser(abc.ABC):
       results.append(self.parse_row(result))
     return results
 
-  @abc.abstractmethod
-  def parse_row(self, row):
-    """Parses single row from response."""
-
-
-class ListParser(BaseParser):
-  """Returns API results as is."""
+  def _evalute_virtual_column(
+    self,
+    fields: list[str],
+    virtual_column_values: dict[str, Any],
+    substitute_expression: str,
+  ) -> api_clients.ApiRowElement:
+    virtual_column_replacements = {
+      field.replace('.', '_'): value
+      for field, value in zip(fields, virtual_column_values)
+    }
+    virtual_column_expression = substitute_expression.format(
+      **virtual_column_replacements
+    )
+    try:
+      tree = ast.parse(virtual_column_expression, mode='eval')
+      valid = all(
+        isinstance(node, VALID_VIRTUAL_COLUMN_OPERATORS)
+        for node in ast.walk(tree)
+      )
+      if valid:
+        return eval(
+          compile(tree, filename='', mode='eval'), {'__builtins__': None}
+        )
+    except ZeroDivisionError:
+      return 0
+    return None
+
+  def process_virtual_column(
+    self,
+    row: api_clients.ApiResponseRow,
+    virtual_column: query_editor.VirtualColumn,
+  ) -> api_clients.ApiRowElement:
+    if virtual_column.type == 'built-in':
+      return virtual_column.value
+    virtual_column_values = [
+      self.parse_row_element(row, field) for field in virtual_column.fields
+    ]
+    try:
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except TypeError:
+      virtual_column_values = [
+        f"'{self.parse_row_element(row, field)}'"
+        for field in virtual_column.fields
+      ]
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except SyntaxError:
+      return virtual_column.value
+    return result
 
-  @override
   def parse_row(
     self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    return row
+    row: api_clients.ApiResponseRow,
+  ) -> list[api_clients.ApiRowElement]:
+    """Parses single row from response."""
+    results = []
+    fields = self.query_spec.fields
+    index = 0
+    for column in self.query_spec.column_names:
+      if virtual_column := self.query_spec.virtual_columns.get(column):
+        result = self.process_virtual_column(row, virtual_column)
+      else:
+        result = self.parse_row_element(row, fields[index])
+        index = index + 1
+      results.append(result)
+    return results
+
+  @abc.abstractmethod
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
 
 
 class DictParser(BaseParser):
   """Extracts nested dict elements."""
 
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
     if not isinstance(row, Mapping):
       raise GarfParserError
-    result = []
-    for field in self.query_spec.fields:
-      result.append(self.get_nested_field(row, field))
-    return result
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Returns nested fields from a dictionary."""
-    if result := dictionary.get(key):
+    if result := row.get(key):
       return result
     key = key.split('.')
     try:
-      return functools.reduce(operator.getitem, key, dictionary)
+      return functools.reduce(operator.getitem, key, row)
     except (TypeError, KeyError):
       return None
 
@@ -93,7 +157,9 @@ class DictParser(BaseParser):
 class NumericConverterDictParser(DictParser):
   """Extracts nested dict elements with numerical conversions."""
 
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
     """Extract nested field with int/float conversion."""
 
     def convert_field(value):
@@ -102,12 +168,12 @@ class NumericConverterDictParser(DictParser):
           return type_(value)
       return value
 
-    if result := dictionary.get(key):
+    if result := row.get(key):
       return convert_field(result)
 
     key = key.split('.')
     try:
-      field = functools.reduce(operator.getitem, key, dictionary)
+      field = functools.reduce(operator.getitem, key, row)
       if isinstance(field, MutableSequence) or field in (True, False):
         return field
       return convert_field(field)

garf_core/query_editor.py

@@ -29,7 +29,7 @@ from typing_extensions import Self, TypeAlias
 
 from garf_core import exceptions
 
-QueryParameters: TypeAlias = dict[str, Union[str, float, int]]
+QueryParameters: TypeAlias = dict[str, Union[str, float, int, list]]
 
 
 class GarfQueryParameters(pydantic.BaseModel):
@@ -125,6 +125,8 @@ class ProcessedField(pydantic.BaseModel):
 
   @classmethod
   def _extract_nested_resource(cls, line_elements: str) -> list[str]:
+    if '://' in line_elements:
+      return []
     return re.split(':', line_elements)
 
 
@@ -160,22 +162,29 @@ class VirtualColumn:
       return VirtualColumn(type='built-in', value=field)
 
     operators = ('/', r'\*', r'\+', ' - ')
-    if len(expressions := re.split('|'.join(operators), field)) > 1:
+    if '://' in field:
+      expressions = re.split(r'\+', field)
+    else:
+      expressions = re.split('|'.join(operators), field)
+    if len(expressions) > 1:
       virtual_column_fields = []
       substitute_expression = field
       for expression in expressions:
         element = expression.strip()
-        if True:
-          # if self._is_valid_field(element):
+        if not _is_constant(element):
           virtual_column_fields.append(element)
           substitute_expression = substitute_expression.replace(
             element, f'{{{element}}}'
           )
+      pattern = r'\{([^}]*)\}'
+      substitute_expression = re.sub(
+        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
+      )
       return VirtualColumn(
         type='expression',
         value=field.format(**macros) if macros else field,
         fields=virtual_column_fields,
-        substitute_expression=substitute_expression.replace('.', '_'),
+        substitute_expression=substitute_expression,
       )
     if not _is_quoted_string(field):
       raise GarfFieldError(f"Incorrect field '{field}'.")
@@ -211,7 +220,11 @@ class ExtractedLineElements:
     field, *alias = re.split(' [Aa][Ss] ', line)
     processed_field = ProcessedField.from_raw(field)
     field = processed_field.field
-    virtual_column = None if field else VirtualColumn.from_raw(field, macros)
+    virtual_column = (
+      VirtualColumn.from_raw(field, macros)
+      if _is_invalid_field(field)
+      else None
+    )
     if alias and processed_field.customizer_type:
       customizer = {
         'type': processed_field.customizer_type,
@@ -373,7 +386,7 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
     self,
     text: str,
     title: str | None = None,
-    args: GarfQueryParameters | None = GarfQueryParameters(),
+    args: GarfQueryParameters | None = None,
     **kwargs: str,
   ) -> None:
     """Instantiates QuerySpecification based on text, title and optional args.
@@ -392,7 +405,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
     """Returns macros with injected common parameters."""
     common_params = dict(self.common_params)
     if macros := self.args.macro:
-      common_params.update(macros)
+      converted_macros = {
+        key: convert_date(value) for key, value in macros.items()
+      }
+      common_params.update(converted_macros)
     return common_params
 
   def generate(self) -> BaseQueryElements:
@@ -435,10 +451,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       if re.match('/\\*', line) or multiline_comment:
         multiline_comment = True
         continue
-      if re.match('^(#|--|//)', line):
+      if re.match('^(#|--|//) ', line):
         continue
       cleaned_query_line = re.sub(
-        ';$', '', re.sub('(--|//).*$', '', line).strip()
+        ';$', '', re.sub('(--|//) .*$', '', line).strip()
       )
       result.append(cleaned_query_line)
     self.query.text = ' '.join(result)
@@ -506,6 +522,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       line_elements = ExtractedLineElements.from_query_line(line)
       if virtual_column := line_elements.virtual_column:
         self.query.virtual_columns[line_elements.alias] = virtual_column
+        if fields := virtual_column.fields:
+          for field in fields:
+            if field not in self.query.fields:
+              self.query.fields.append(field)
     return self
 
   def extract_customizers(self) -> Self:
@@ -533,3 +553,64 @@ def _is_quoted_string(field_name: str) -> bool:
   return (field_name.startswith("'") and field_name.endswith("'")) or (
     field_name.startswith('"') and field_name.endswith('"')
   )
+
+
+def _is_constant(element) -> bool:
+  with contextlib.suppress(ValueError):
+    float(element)
+    return True
+  return _is_quoted_string(element)
+
+
+def _is_invalid_field(field) -> bool:
+  operators = ('/', '*', '+', ' - ')
+  is_constant = _is_constant(field)
+  has_operator = any(operator in field for operator in operators)
+  return is_constant or has_operator
+
+
+def convert_date(date_string: str) -> str:
+  """Converts specific dates parameters to actual dates.
+
+  Returns:
+    Date string in YYYY-MM-DD format.
+
+  Raises:
+    GarfMacroError:
+     If dynamic lookback value (:YYYYMMDD-N) is incorrect.
+  """
+  if isinstance(date_string, list) or date_string.find(':Y') == -1:
+    return date_string
+  current_date = datetime.date.today()
+  base_date, *date_customizer = re.split('\\+|-', date_string)
+  if len(date_customizer) > 1:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+  if not date_customizer:
+    days_lookback = 0
+  else:
+    try:
+      days_lookback = int(date_customizer[0])
+    except ValueError as e:
+      raise GarfMacroError(
+        'Must provide numeric value for a number lookback period, '
+        'i.e. :YYYYMMDD-1'
+      ) from e
+  if base_date == ':YYYY':
+    new_date = datetime.datetime(current_date.year, 1, 1)
+    delta = relativedelta.relativedelta(years=days_lookback)
+  elif base_date == ':YYYYMM':
+    new_date = datetime.datetime(current_date.year, current_date.month, 1)
+    delta = relativedelta.relativedelta(months=days_lookback)
+  elif base_date == ':YYYYMMDD':
+    new_date = current_date
+    delta = relativedelta.relativedelta(days=days_lookback)
+  else:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+
+  if '-' in date_string:
+    return (new_date - delta).strftime('%Y-%m-%d')
+  return (new_date + delta).strftime('%Y-%m-%d')

garf_core/report_fetcher.py

@@ -54,16 +54,18 @@ class ApiReportFetcher:
   """Class responsible for getting data from report API.
 
   Attributes:
-      api_client: a client used for connecting to API.
-      parser: Type of parser to convert API response.
+      api_client: Client used for connecting to API.
+      parser: Class of parser to convert API response.
       query_specification_builder: Class to perform query parsing.
+      builtin_queries:
+        Mapping between query name and function for generating GarfReport.
   """
 
   def __init__(
     self,
-    api_client: api_clients.BaseApiClient,
-    parser: parsers.BaseParser = parsers.DictParser,
-    query_specification_builder: query_editor.QuerySpecification = (
+    api_client: api_clients.BaseClient,
+    parser: type[parsers.BaseParser] = parsers.DictParser,
+    query_specification_builder: type[query_editor.QuerySpecification] = (
       query_editor.QuerySpecification
     ),
     builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]]
@@ -94,7 +96,7 @@ class ApiReportFetcher:
 
   async def afetch(
     self,
-    query_specification: str | query_editor.QueryElements,
+    query_specification: str | query_editor.QuerySpecification,
     args: query_editor.GarfQueryParameters | None = None,
     **kwargs: str,
   ) -> report.GarfReport:

{garf_core-0.1.5.post0.dist-info → garf_core-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.1.5.post0
+Version: 0.2.1
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

garf_core-0.2.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+garf_core/__init__.py,sha256=ml6Xq1xyIO0Mx2FJsW0lTQ7z51UhpmMWfANHqedkpQg,954
+garf_core/api_clients.py,sha256=PTuNz-nsnAwAhh1eyqt6fx0O3kbnL_1W5lxU_3YmhF4,4747
+garf_core/base_query.py,sha256=ZDAw2ojmismXRO0HXEvKDukpS7OAc7390LnM8kvCSCY,1201
+garf_core/exceptions.py,sha256=Gzvkl2M-rA_XQRAMd3CC62KHeFQE_b6uby0fD0pouw4,1269
+garf_core/parsers.py,sha256=Y1hJFvvtRCiiR0A-Vh8fIVjBVLdmKZEnxRnGJ5I7hZw,5353
+garf_core/query_editor.py,sha256=0bOX_Tz3D3KCcUrNddp_do4aoWAd0IEBtM-1u7JcJVU,19603
+garf_core/report.py,sha256=J2-oNV3Nh7ToYhm2fsdovkiE__-UhT0K5-wP0jBD0oQ,20330
+garf_core/report_fetcher.py,sha256=478HrA31-mCEP2E6cPlxonvBrddBBnSUxFp5D3yE-yU,4994
+garf_core/fetchers/__init__.py,sha256=_cSjg1D5RhUKxaVeVbaDdb8AAoI9glKJXgN5H4qXFkw,783
+garf_core/fetchers/fake.py,sha256=fgJjxuHyd6EIUflUtj8r_HfaMS1YTDrOqDlaj6Kvbjs,2584
+garf_core/fetchers/rest.py,sha256=-5B2-Ck_t3hG99ym59AKwlzctiDxRFI2Nnc8STBxRDo,2201
+garf_core-0.2.1.dist-info/METADATA,sha256=0wuvUWUnnoqwNI9Veg2BNeOAncAcB4BOwTKmPf9CmeM,2427
+garf_core-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+garf_core-0.2.1.dist-info/entry_points.txt,sha256=u4h-ujHO1hbxVXRQzwcC4ftju9_KBYtq5mCLKEBHMj0,69
+garf_core-0.2.1.dist-info/top_level.txt,sha256=Gj-Zp7fM2turGut5vTJuo5xEcKfow7cTLX3y3WFfNgA,10
+garf_core-0.2.1.dist-info/RECORD,,

garf_core-0.1.5.post0.dist-info/RECORD

@@ -1,16 +0,0 @@
-garf_core/__init__.py,sha256=ZKDzKwZl1HtqUnBH6ijUMZgu6dRZJsbgMGuL8s9gzNU,960
-garf_core/api_clients.py,sha256=P9fSy0xUXqmQoupWGWP1-SVBz_yZ80Jh58fvkh2dY8s,4694
-garf_core/base_query.py,sha256=ZDAw2ojmismXRO0HXEvKDukpS7OAc7390LnM8kvCSCY,1201
-garf_core/exceptions.py,sha256=Gzvkl2M-rA_XQRAMd3CC62KHeFQE_b6uby0fD0pouw4,1269
-garf_core/parsers.py,sha256=A4Yb614ydN_psKvyUfWZjnTG5-aooT0pz2Cm6CHUnC4,3283
-garf_core/query_editor.py,sha256=QPbUz8jHloTcGm_Nz2Nog_sdYIPiDshtm_t45-R76pk,17147
-garf_core/report.py,sha256=J2-oNV3Nh7ToYhm2fsdovkiE__-UhT0K5-wP0jBD0oQ,20330
-garf_core/report_fetcher.py,sha256=dgErrkRv1bdPN0npX3atZbJrOpzy9OmoPC38e2FpBSQ,4883
-garf_core/fetchers/__init__.py,sha256=_cSjg1D5RhUKxaVeVbaDdb8AAoI9glKJXgN5H4qXFkw,783
-garf_core/fetchers/fake.py,sha256=fgJjxuHyd6EIUflUtj8r_HfaMS1YTDrOqDlaj6Kvbjs,2584
-garf_core/fetchers/rest.py,sha256=-5B2-Ck_t3hG99ym59AKwlzctiDxRFI2Nnc8STBxRDo,2201
-garf_core-0.1.5.post0.dist-info/METADATA,sha256=-YyPa7wDx_3dzKaWEPlEupNyg2QtKblNLvem6pWhew0,2433
-garf_core-0.1.5.post0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-garf_core-0.1.5.post0.dist-info/entry_points.txt,sha256=u4h-ujHO1hbxVXRQzwcC4ftju9_KBYtq5mCLKEBHMj0,69
-garf_core-0.1.5.post0.dist-info/top_level.txt,sha256=Gj-Zp7fM2turGut5vTJuo5xEcKfow7cTLX3y3WFfNgA,10
-garf_core-0.1.5.post0.dist-info/RECORD,,