keboola.component 1.6.1__tar.gz → 1.6.4__tar.gz

{keboola.component-1.6.1/src/keboola.component.egg-info → keboola.component-1.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.1
+Version: 1.6.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team
@@ -231,6 +231,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python

{keboola.component-1.6.1 → keboola.component-1.6.4}/README.md

@@ -212,6 +212,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python

{keboola.component-1.6.1 → keboola.component-1.6.4}/setup.py

@@ -13,7 +13,7 @@ project_urls = {
 
 setuptools.setup(
     name="keboola.component",
-    version="1.6.1",
+    version="1.6.4",
     author="Keboola KDS Team",
     project_urls=project_urls,
     setup_requires=['pytest-runner', 'flake8'],

{keboola.component-1.6.1 → keboola.component-1.6.4}/src/keboola/component/dao.py

@@ -582,6 +582,7 @@ class SupportedManifestAttributes(SubscriptableDataclass):
         elif stage == 'in':
             attributes = self.in_attributes
             exclude = self.in_legacy_exclude
+
         else:
             raise ValueError(f'Unsupported stage {stage}')
 
@@ -597,9 +598,6 @@ class IODefinition(ABC):
     def __init__(self, full_path):
         self.full_path = full_path
 
-        # infer stage by default
-        self.__stage = self.__get_stage_inferred()
-
     @classmethod
     def build_from_manifest(cls,
                             manifest_file_path: str
@@ -619,13 +617,13 @@ class IODefinition(ABC):
         """
         Helper property marking the stage of the file. (str)
         """
-        return self.__stage
+        return self._stage
 
     @stage.setter
     def stage(self, stage: str):
         if stage not in ['in', 'out']:
             raise ValueError(f'Invalid stage "{stage}", supported values are: "in", "out"')
-        self.__stage = stage
+        self._stage = stage
 
     @property
     @abstractmethod
@@ -643,17 +641,6 @@ class IODefinition(ABC):
         """
         raise NotImplementedError
 
-    def __get_stage_inferred(self):
-        stage = 'in'
-        if not self.full_path or not Path(self.full_path).exists():
-            return stage
-
-        if Path(self.full_path).parent.parent.name == 'in':
-            stage = 'in'
-        elif Path(self.full_path).parent.parent.name == 'out':
-            stage = 'out'
-        return stage
-
     # ############ Staging parameters
 
     @dataclass
@@ -757,6 +744,11 @@ class TableDefinition(IODefinition):
         "columns",
         "metadata",
         "column_metadata",
+        "rows_count",
+        "data_size_bytes",
+        "is_alias",
+        "attributes",
+        "indexed_columns"
     ]
 
     OUTPUT_MANIFEST_ATTRIBUTES = [
@@ -792,7 +784,7 @@ class TableDefinition(IODefinition):
                  enclosure: Optional[str] = '"',
                  delimiter: Optional[str] = ',',
                  delete_where: Optional[dict] = None,
-                 stage: Optional[str] = None,
+                 stage: Optional[str] = 'out',
                  write_always: Optional[bool] = False,
                  has_header: Optional[bool] = None,
                  # input
@@ -869,8 +861,10 @@ class TableDefinition(IODefinition):
         self._rows_count = kwargs.get('rows_count')
         self._data_size_bytes = kwargs.get('data_size_bytes')
         self._is_alias = kwargs.get('is_alias')
+        self._indexed_columns = kwargs.get('indexed_columns')
+        self._attributes = kwargs.get('attributes')
 
-        self.stage = stage if stage else self.__get_stage_inferred()
+        self.stage = stage
         self.has_header = has_header or self._has_header_in_file()
 
     def __get_stage_inferred(self):
@@ -1101,7 +1095,13 @@ class TableDefinition(IODefinition):
                           DeprecationWarning)
             force_legacy_mode = True
 
+        if manifest.get('id'):
+            stage = 'in'
+        else:
+            stage = 'out'
+
         table_def = cls(name=name,
+                        stage=stage,
                         full_path=full_path,
                         is_sliced=is_sliced,
                         id=manifest.get('id'),
@@ -1115,12 +1115,14 @@ class TableDefinition(IODefinition):
                         rows_count=manifest.get('rows_count'),
                         data_size_bytes=manifest.get('data_size_bytes'),
                         is_alias=manifest.get('is_alias'),
-                        force_legacy_mode=force_legacy_mode
+                        force_legacy_mode=force_legacy_mode,
+                        indexed_columns=manifest.get('indexed_columns'),
+                        attributes=manifest.get('attributes')
                         )
 
         return table_def
 
-    def get_manifest_dictionary(self, manifest_type: Optional[str] = 'out', legacy_queue: bool = False,
+    def get_manifest_dictionary(self, manifest_type: Optional[str] = None, legacy_queue: bool = False,
                                 legacy_manifest: Optional[bool] = None) -> dict:
         """
         Returns manifest dictionary in appropriate manifest_type: either 'in' or 'out'.
@@ -1192,6 +1194,8 @@ class TableDefinition(IODefinition):
             'rows_count': self._rows_count,
             'data_size_bytes': self._data_size_bytes,
             'is_alias': self._is_alias,
+            'indexed_columns': self._indexed_columns,
+            'attributes': self._attributes,
 
             'destination': self.destination,
             'incremental': self.incremental,
@@ -1395,6 +1399,8 @@ class TableDefinition(IODefinition):
         self._table_metadata = table_metadata
         # backward compatibility legacy support
         for col, val in table_metadata._get_legacy_column_metadata_for_manifest().items():
+            if not self.schema.get(col):
+                self.schema[col] = ColumnDefinition()
             self.schema[col].metadata = {item['key']: item['value'] for item in val}
 
     @property
@@ -1536,6 +1542,7 @@ class FileDefinition(IODefinition):
                             "notify"]
 
     def __init__(self, full_path: str,
+                 stage: Optional[str] = 'out',
                  tags: Optional[List[str]] = None,
                  is_public: Optional[bool] = False,
                  is_permanent: Optional[bool] = False,
@@ -1552,6 +1559,7 @@ class FileDefinition(IODefinition):
 
         Args:
             full_path (str): Full path of the file.
+            stage (str): Storage Stage 'in' or 'out' default out
             tags (list):
                 List of tags that are assigned to this file
             is_public: When true, the file URL will be permanent and publicly accessible.
@@ -1562,6 +1570,8 @@ class FileDefinition(IODefinition):
         """
         super().__init__(full_path)
 
+        self.stage = stage
+
         self.tags = tags
         self.is_public = is_public
         self.is_permanent = is_permanent
@@ -1602,7 +1612,7 @@ class FileDefinition(IODefinition):
         Returns:
             An instance of FileDefinition configured for output files.
         """
-        return cls(full_path=full_path, tags=tags, is_public=is_public, is_permanent=is_permanent,
+        return cls(full_path=full_path, stage="out", tags=tags, is_public=is_public, is_permanent=is_permanent,
                    is_encrypted=is_encrypted, notify=notify)
 
     @classmethod
@@ -1632,7 +1642,7 @@ class FileDefinition(IODefinition):
         Returns:
             An instance of FileDefinition configured for input files.
         """
-        return cls(full_path=full_path, id=id, s3=s3, abs=abs, created=created, size_bytes=size_bytes,
+        return cls(full_path=full_path, stage="in", id=id, s3=s3, abs=abs, created=created, size_bytes=size_bytes,
                    max_age_days=max_age_days)
 
     @classmethod
@@ -1667,7 +1677,13 @@ class FileDefinition(IODefinition):
 
         full_path = str(file_path)
 
+        if manifest.get('id'):
+            stage = 'in'
+        else:
+            stage = 'out'
+
         file_def = cls(full_path=full_path,
+                       stage=stage,
                        tags=manifest.get('tags', []),
                        is_public=manifest.get('is_public', False),
                        is_permanent=manifest.get('is_permanent', False),
@@ -1732,18 +1748,31 @@ class FileDefinition(IODefinition):
 
         """
 
-        return {
-            'id': self.id,
-            'created': self.created.strftime('%Y-%m-%dT%H:%M:%S%z') if self.created else None,
-            'is_public': self.is_public,
-            'is_encrypted': self.is_encrypted,
-            'name': self.name,
-            'size_bytes': self.size_bytes,
-            'tags': self.tags,
-            'notify': self.notify,
-            'max_age_days': self.max_age_days,
-            'is_permanent': self.is_permanent,
-        }
+        if manifest_type == 'in':
+
+            manifest_dictionary = {
+                'id': self.id,
+                'created': self.created.strftime('%Y-%m-%dT%H:%M:%S%z') if self.created else None,
+                'is_public': self.is_public,
+                'is_encrypted': self.is_encrypted,
+                'name': self.name,
+                'size_bytes': self.size_bytes,
+                'tags': self.tags,
+                'notify': self.notify,
+                'max_age_days': self.max_age_days,
+                'is_permanent': self.is_permanent,
+            }
+
+        else:
+            manifest_dictionary = {
+                'is_public': self.is_public,
+                'is_permanent': self.is_permanent,
+                'is_encrypted': self.is_encrypted,
+                'tags': self.tags,
+                'notify': self.notify,
+            }
+
+        return manifest_dictionary
 
     @property
     def name(self) -> str:

{keboola.component-1.6.1 → keboola.component-1.6.4}/src/keboola/component/interface.py

@@ -950,7 +950,8 @@ class CommonInterface:
         """
 
         if not legacy_manifest:
-            legacy_manifest = self.environment_variables.data_type_support not in ('authoritative', 'hints')
+
+            legacy_manifest = self._expects_legacy_manifest()
 
         manifest = io_definition.get_manifest_dictionary(legacy_queue=self.is_legacy_queue,
                                                          legacy_manifest=legacy_manifest)
@@ -959,6 +960,13 @@ class CommonInterface:
         with open(io_definition.full_path + '.manifest', 'w') as manifest_file:
             json.dump(manifest, manifest_file)
 
+    def _expects_legacy_manifest(self) -> bool:
+        legacy_manifest = self.environment_variables.data_type_support not in ('authoritative', 'hints')
+        om_override = self.configuration.config_data.get('storage', {}).get('output', {}).get('data_type_support')
+        if om_override:
+            legacy_manifest = om_override not in ('authoritative', 'hints')
+        return legacy_manifest
+
     def write_manifests(self, io_definitions: List[Union[dao.FileDefinition, dao.TableDefinition]],
                         legacy_manifest: Optional[bool] = None):
         """

{keboola.component-1.6.1 → keboola.component-1.6.4/src/keboola.component.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.1
+Version: 1.6.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team
@@ -231,6 +231,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python