keboola.component 1.4.2__tar.gz → 1.4.4__tar.gz

{keboola.component-1.4.2/src/keboola.component.egg-info → keboola.component-1.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.4.2
+Version: 1.4.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team

{keboola.component-1.4.2 → keboola.component-1.4.4}/setup.py

@@ -13,7 +13,7 @@ project_urls = {
 
 setuptools.setup(
     name="keboola.component",
-    version="1.4.2",
+    version="1.4.4",
     author="Keboola KDS Team",
     project_urls=project_urls,
     setup_requires=['pytest-runner', 'flake8'],

{keboola.component-1.4.2 → keboola.component-1.4.4}/src/keboola/component/dao.py

@@ -1,5 +1,6 @@
 import dataclasses
 import json
+import logging
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
@@ -7,6 +8,11 @@ from enum import Enum
 from pathlib import Path
 from typing import List, Union, Dict, Optional
 
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal
+
 KBC_DEFAULT_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S%z'
 
 
@@ -381,6 +387,30 @@ class TableMetadata:
             raise ValueError(', '.join(errors) + f'\n Supported base types are: [{SupportedDataTypes.list()}]')
 
 
+@dataclass
+class SupportedManifestAttributes(SubscriptableDataclass):
+    out_attributes: List[str]
+    in_attributes: List[str]
+    out_legacy_exclude: List[str] = dataclasses.field(default_factory=lambda: [])
+    in_legacy_exclude: List[str] = dataclasses.field(default_factory=lambda: [])
+
+    def get_attributes_by_stage(self, stage: Literal["in", "out"], legacy_queue: bool = False) -> List[str]:
+        if stage == 'out':
+            attributes = self.out_attributes
+            exclude = self.out_legacy_exclude
+        elif stage == 'in':
+            attributes = self.in_attributes
+            exclude = self.in_legacy_exclude
+        else:
+            raise ValueError(f"Unsupported stage {stage}")
+
+        if legacy_queue:
+            logging.warning(f"Running on legacy queue some manifest properties will be ignored: {exclude}")
+            attributes = list(set(attributes).difference(exclude))
+
+        return attributes
+
+
 class IODefinition(ABC):
 
     def __init__(self, full_path):
@@ -396,7 +426,7 @@ class IODefinition(ABC):
                             ):
         raise NotImplementedError
 
-    def _filter_attributes_by_manifest_type(self, manifest_type):
+    def _filter_attributes_by_manifest_type(self, manifest_type: Literal["in", "out"], legacy_queue: bool = False):
         """
         Filter manifest to contain only supported fields
         Args:
@@ -405,7 +435,8 @@ class IODefinition(ABC):
         Returns:
 
         """
-        supported_fields = self._manifest_attributes.get(manifest_type, [])
+        supported_fields = self._manifest_attributes.get_attributes_by_stage(manifest_type, legacy_queue)
+
         new_dict = self._raw_manifest.copy()
         if supported_fields:
             for attr in self._raw_manifest:
@@ -413,10 +444,10 @@ class IODefinition(ABC):
                     new_dict.pop(attr, None)
         return new_dict
 
-    def get_manifest_dictionary(self, manifest_type: Optional[str] = None) -> dict:
+    def get_manifest_dictionary(self, manifest_type: Optional[str] = None, legacy_queue=False) -> dict:
         """
         Returns manifest dictionary in appropriate manifest_type: either 'in' or 'out'.
-        By default returns output manifest.
+        By default, returns output manifest.
              The result keeps only values that are applicable for
              the selected type of the Manifest file. Because although input and output manifests share most of
              the attributes, some are not shared.
@@ -428,6 +459,7 @@ class IODefinition(ABC):
             manifest_type (str): either 'in' or 'out'.
              See [manifest files](https://developers.keboola.com/extend/common-interface/manifest-files)
              for more information.
+            legacy_queue (bool): optional flag marking project on legacy queue.(some options are not allowed on queue2)
 
         Returns:
             dict representation of the manifest file in a format expected / produced by the Keboola Connection
@@ -436,7 +468,7 @@ class IODefinition(ABC):
         if not manifest_type:
             manifest_type = self.stage
 
-        return self._filter_attributes_by_manifest_type(manifest_type)
+        return self._filter_attributes_by_manifest_type(manifest_type, legacy_queue)
 
     @property
     def stage(self) -> str:
@@ -453,11 +485,11 @@ class IODefinition(ABC):
 
     @property
     @abstractmethod
-    def _manifest_attributes(self) -> Dict[str, List[str]]:
+    def _manifest_attributes(self) -> SupportedManifestAttributes:
         """
         Manifest attributes
         """
-        return {}
+        return SupportedManifestAttributes([], [])
 
     @property
     @abstractmethod
@@ -587,6 +619,7 @@ class TableDefinition(IODefinition):
         "columns",
         "incremental",
         "primary_key",
+        "write_always",
         "delimiter",
         "enclosure",
         "metadata",
@@ -596,6 +629,10 @@ class TableDefinition(IODefinition):
         "delete_where_operator"
     ]
 
+    OUTPUT_MANIFEST_LEGACY_EXCLUDES = [
+        "write_always"
+    ]
+
     MANIFEST_ATTRIBUTES = {'in': INPUT_MANIFEST_ATTRIBUTES,
                            'out': OUTPUT_MANIFEST_ATTRIBUTES}
 
@@ -608,7 +645,9 @@ class TableDefinition(IODefinition):
                  enclosure: str = '"',
                  delimiter: str = ',',
                  delete_where: dict = None,
-                 stage: str = 'in'):
+                 stage: str = 'in',
+                 write_always: bool = False
+                 ):
         """
 
         Args:
@@ -630,6 +669,8 @@ class TableDefinition(IODefinition):
             delimiter: str: CSV delimiter, by default ,
             delete_where (dict): Dict with settings for deleting rows
             stage: str: Storage Stage 'in' or 'out'
+            write_always: Bool: If true, the table will be saved to Storage even when the job execution
+                           fails.
         """
         super().__init__(full_path)
         self._name = name
@@ -650,6 +691,7 @@ class TableDefinition(IODefinition):
         self.table_metadata = table_metadata
         self.set_delete_where_from_dict(delete_where)
         self.stage = stage
+        self.write_always = write_always
 
     @classmethod
     def build_from_manifest(cls,
@@ -705,8 +747,9 @@ class TableDefinition(IODefinition):
         return table_def
 
     @property
-    def _manifest_attributes(self) -> Dict[str, List[str]]:
-        return self.MANIFEST_ATTRIBUTES
+    def _manifest_attributes(self) -> SupportedManifestAttributes:
+        return SupportedManifestAttributes(self.MANIFEST_ATTRIBUTES['out'], self.MANIFEST_ATTRIBUTES['in'],
+                                           self.OUTPUT_MANIFEST_LEGACY_EXCLUDES)
 
     # #### Manifest properties
     @property
@@ -797,6 +840,14 @@ class TableDefinition(IODefinition):
         if incremental:
             self._raw_manifest['incremental'] = True
 
+    @property
+    def write_always(self) -> bool:
+        return self._raw_manifest.get('write_always', False)
+
+    @write_always.setter
+    def write_always(self, write_always: bool):
+        self._raw_manifest['write_always'] = write_always
+
     @property
     def primary_key(self) -> List[str]:
         return self._raw_manifest.get('primary_key', [])
@@ -864,7 +915,7 @@ class TableDefinition(IODefinition):
         self._raw_manifest['metadata'] = table_metadata.get_table_metadata_for_manifest()
         self._raw_manifest['column_metadata'] = table_metadata.get_column_metadata_for_manifest()
 
-    def get_manifest_dictionary(self, stage_type: Optional[str] = None) -> dict:
+    def get_manifest_dictionary(self, stage_type: Optional[str] = None, legacy_queue=False) -> dict:
         """
 
         Args:
@@ -877,8 +928,8 @@ class TableDefinition(IODefinition):
         """
         # in case the table_metadata is out of sync, e.g. the object was modified in-place
         self._set_table_metadata_to_manifest(self._table_metadata)
-        self._raw_manifest = super(TableDefinition, self).get_manifest_dictionary(stage_type)
-        return self._raw_manifest
+        raw_manifest = super(TableDefinition, self).get_manifest_dictionary(stage_type, legacy_queue)
+        return raw_manifest
 
 
 class FileDefinition(IODefinition):
@@ -1020,8 +1071,8 @@ class FileDefinition(IODefinition):
         return Path(self.full_path).name
 
     @property
-    def _manifest_attributes(self) -> Dict[str, List[str]]:
-        return {'out': self.OUTPUT_MANIFEST_KEYS}
+    def _manifest_attributes(self) -> SupportedManifestAttributes:
+        return SupportedManifestAttributes(self.OUTPUT_MANIFEST_KEYS, [])
 
     # ########### Output manifest properties - R/W
 

{keboola.component-1.4.2 → keboola.component-1.4.4}/src/keboola/component/interface.py

@@ -4,11 +4,11 @@ import glob
 import json
 import logging
 import os
+import sys
 from datetime import datetime
 from pathlib import Path
 from typing import List, Dict, Optional, Union
 
-import sys
 from deprecated import deprecated
 from pygelf import GelfUdpHandler, GelfTcpHandler
 from pytz import utc
@@ -327,7 +327,8 @@ class CommonInterface:
                                  table_metadata: dao.TableMetadata = None,
                                  enclosure: str = '"',
                                  delimiter: str = ',',
-                                 delete_where: dict = None) -> dao.TableDefinition:
+                                 delete_where: dict = None,
+                                 write_always: bool = False) -> dao.TableDefinition:
         """
                 Helper method for dao.TableDefinition creation along with the "manifest".
                 It initializes path according to the storage_stage type.
@@ -347,6 +348,8 @@ class CommonInterface:
                     enclosure: str: CSV enclosure, by default "
                     delimiter: str: CSV delimiter, by default ,
                     delete_where: Dict with settings for deleting rows
+                    write_always: Bool: If true, the table will be saved to Storage even when the job execution
+                           fails.
         """
         if storage_stage == 'in':
             full_path = os.path.join(self.tables_in_path, name)
@@ -366,7 +369,8 @@ class CommonInterface:
                                    enclosure=enclosure,
                                    delimiter=delimiter,
                                    delete_where=delete_where,
-                                   stage=storage_stage)
+                                   stage=storage_stage,
+                                   write_always=write_always)
 
     def create_in_table_definition(self, name: str,
                                    is_sliced: bool = False,
@@ -410,7 +414,8 @@ class CommonInterface:
                                     table_metadata: dao.TableMetadata = None,
                                     enclosure: str = '"',
                                     delimiter: str = ',',
-                                    delete_where: dict = None) -> dao.TableDefinition:
+                                    delete_where: dict = None,
+                                    write_always: bool = False) -> dao.TableDefinition:
         """
                        Helper method for output dao.TableDefinition creation along with the "manifest".
                        It initializes path in data/tables/out/ folder.
@@ -426,6 +431,8 @@ class CommonInterface:
                            enclosure: str: CSV enclosure, by default "
                            delimiter: str: CSV delimiter, by default ,
                            delete_where: Dict with settings for deleting rows
+                           write_always: Bool: If true, the table will be saved to Storage even when the job execution
+                           fails.
         """
 
         return self._create_table_definition(name=name,
@@ -438,7 +445,8 @@ class CommonInterface:
                                              table_metadata=table_metadata,
                                              enclosure=enclosure,
                                              delimiter=delimiter,
-                                             delete_where=delete_where)
+                                             delete_where=delete_where,
+                                             write_always=write_always)
 
     # # File processing
 
@@ -862,8 +870,20 @@ class CommonInterface:
     def files_in_path(self):
         return os.path.join(self.data_folder_path, 'in', 'files')
 
-    @staticmethod
-    def write_manifest(io_definition: Union[dao.FileDefinition, dao.TableDefinition]):
+    @property
+    def is_legacy_queue(self) -> bool:
+        """
+        Check if the project is running on legacy queue (v1)
+        Returns:
+
+        """
+        features = os.environ.get('KBC_PROJECT_FEATURE_GATES')
+        is_legacy_queue = True
+        if not features or 'queuev2' in features:
+            is_legacy_queue = False
+        return is_legacy_queue
+
+    def write_manifest(self, io_definition: Union[dao.FileDefinition, dao.TableDefinition]):
         """
         Write a table manifest from dao.IODefinition. Creates the appropriate manifest file in the proper location.
 
@@ -895,14 +915,14 @@ class CommonInterface:
         Returns:
 
         """
-        manifest = io_definition.get_manifest_dictionary()
+
+        manifest = io_definition.get_manifest_dictionary(legacy_queue=self.is_legacy_queue)
         # make dirs if not exist
         os.makedirs(os.path.dirname(io_definition.full_path), exist_ok=True)
         with open(io_definition.full_path + '.manifest', 'w') as manifest_file:
             json.dump(manifest, manifest_file)
 
-    @staticmethod
-    def write_manifests(io_definitions: List[Union[dao.FileDefinition, dao.TableDefinition]]):
+    def write_manifests(self, io_definitions: List[Union[dao.FileDefinition, dao.TableDefinition]]):
         """
         Process all table definition objects and create appropriate manifest files.
         Args:
@@ -912,13 +932,12 @@ class CommonInterface:
 
         """
         for io_def in io_definitions:
-            CommonInterface.write_manifest(io_def)
+            self.write_manifest(io_def)
 
     # ############# DEPRECATED METHODS, TODO: remove
 
-    @staticmethod
     @deprecated(version='1.3.0', reason="You should use write_manifest function")
-    def write_filedef_manifest(file_definition: dao.FileDefinition):
+    def write_filedef_manifest(self, file_definition: dao.FileDefinition):
         """
         Write a table manifest from dao.FileDefinition. Creates the appropriate manifest file in the proper location.
 
@@ -942,11 +961,10 @@ class CommonInterface:
         Returns:
 
         """
-        CommonInterface.write_manifest(file_definition)
+        self.write_manifest(file_definition)
 
-    @staticmethod
     @deprecated(version='1.3.0', reason="You should use write_manifests function")
-    def write_filedef_manifests(file_definitions: List[dao.FileDefinition]):
+    def write_filedef_manifests(self, file_definitions: List[dao.FileDefinition]):
         """
         Process all table definition objects and create appropriate manifest files.
         Args:
@@ -955,11 +973,10 @@ class CommonInterface:
         Returns:
 
         """
-        CommonInterface.write_manifests(file_definitions)
+        self.write_manifests(file_definitions)
 
-    @staticmethod
     @deprecated(version='1.3.0', reason="You should use write_manifest function")
-    def write_tabledef_manifest(table_definition: dao.TableDefinition):
+    def write_tabledef_manifest(self, table_definition: dao.TableDefinition):
         """
         Write a table manifest from dao.TableDefinition. Creates the appropriate manifest file in the proper location.
 
@@ -988,11 +1005,10 @@ class CommonInterface:
         Returns:
 
         """
-        CommonInterface.write_manifest(table_definition)
+        self.write_manifest(table_definition)
 
-    @staticmethod
     @deprecated(version='1.3.0', reason="You should use write_manifests function")
-    def write_tabledef_manifests(table_definitions: List[dao.TableDefinition]):
+    def write_tabledef_manifests(self, table_definitions: List[dao.TableDefinition]):
         """
         Process all table definition objects and create appropriate manifest files.
         Args:
@@ -1001,7 +1017,7 @@ class CommonInterface:
         Returns:
 
         """
-        CommonInterface.write_manifests(table_definitions)
+        self.write_manifests(table_definitions)
 
 
 # ########## CONFIGURATION

{keboola.component-1.4.2 → keboola.component-1.4.4}/src/keboola/component/sync_actions.py

@@ -63,7 +63,7 @@ class SelectElement(SyncActionResult):
         self.status = None
 
 
-def process_sync_action_result(result: Union[None, dict, SyncActionResult, List[SyncActionResult]]) -> str:
+def process_sync_action_result(result: Union[None, List[dict], dict, SyncActionResult, List[SyncActionResult]]) -> str:
     """
     Converts Sync Action result into valid string (expected by Sync Action).
     Args:
@@ -75,7 +75,7 @@ def process_sync_action_result(result: Union[None, dict, SyncActionResult, List[
     if isinstance(result, SyncActionResult):
         result_str = str(result)
     elif isinstance(result, list):
-        result_str = f'[{", ".join([str(r) for r in result])}]'
+        result_str = f'[{", ".join([json.dumps(r) if isinstance(r, dict) else str(r) for r in result])}]'
     elif result is None:
         result_str = json.dumps({'status': 'success'})
     elif isinstance(result, dict):

{keboola.component-1.4.2 → keboola.component-1.4.4/src/keboola.component.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.4.2
+Version: 1.4.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team