deltafi 2.30.0__tar.gz → 2.30.1__tar.gz

deltafi-2.30.1/.dockerignore

@@ -0,0 +1,3 @@
+.venv
+venv
+**/__pycache__

deltafi-2.30.1/.gitignore

@@ -0,0 +1,2 @@
+venv
+.venv

deltafi-2.30.1/.python-version

@@ -0,0 +1 @@
+3.13

deltafi-2.30.1/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.4
+Name: deltafi
+Version: 2.30.1
+Summary: SDK for DeltaFi plugins and actions
+Project-URL: Source Code, https://gitlab.com/deltafi/deltafi
+Project-URL: Documentation, https://docs.deltafi.org/#/
+Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
+Author-email: DeltaFi <deltafi@systolic.com>
+License: Apache License, Version 2.0
+Keywords: deltafi
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development
+Requires-Python: >=3.13
+Requires-Dist: deepdiff==6.7.1
+Requires-Dist: json-logging==1.5.1
+Requires-Dist: minio==7.2.15
+Requires-Dist: pydantic==2.11.7
+Requires-Dist: pyyaml==6.0.2
+Requires-Dist: redis==6.2.0
+Requires-Dist: requests==2.32.4
+Requires-Dist: urllib3==2.5.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.8.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: mockito==1.5.4; extra == 'test'
+Requires-Dist: pytest-mock==3.14.1; extra == 'test'
+Requires-Dist: pytest==8.4.1; extra == 'test'
+Description-Content-Type: text/markdown
+
+# DeltaFi Action Kit
+
+This project provides a Python implementation of the DeltaFi Action Kit. The DeltaFi Action Kit is a setup of modules which simplify the creation of a DeltaFi Plugin.
+
+## Development Setup
+
+This project uses [uv](https://github.com/astral-sh/uv) for dependency management and building. uv is a fast Python package installer and resolver.
+
+### Prerequisites
+
+- Python 3.13 or higher
+- uv (will be installed automatically via the build process)
+
+### Building and Testing
+
+The project uses Gradle to orchestrate the build process with uv:
+
+```bash
+# Build the project
+./gradlew assemble
+
+# Run tests
+./gradlew test
+
+# Clean build artifacts
+./gradlew clean
+```
+
+### Manual Development
+
+For manual development without Gradle:
+
+```bash
+cd src
+
+# Install uv if not already installed
+brew install uv || pip install uv
+
+# Install dependencies
+uv pip install -e .[test]
+
+# Run tests
+uv run pytest
+
+# Build package
+uv build
+```

deltafi-2.30.1/README.md

@@ -0,0 +1,47 @@
+# DeltaFi Action Kit
+
+This project provides a Python implementation of the DeltaFi Action Kit. The DeltaFi Action Kit is a setup of modules which simplify the creation of a DeltaFi Plugin.
+
+## Development Setup
+
+This project uses [uv](https://github.com/astral-sh/uv) for dependency management and building. uv is a fast Python package installer and resolver.
+
+### Prerequisites
+
+- Python 3.13 or higher
+- uv (will be installed automatically via the build process)
+
+### Building and Testing
+
+The project uses Gradle to orchestrate the build process with uv:
+
+```bash
+# Build the project
+./gradlew assemble
+
+# Run tests
+./gradlew test
+
+# Clean build artifacts
+./gradlew clean
+```
+
+### Manual Development
+
+For manual development without Gradle:
+
+```bash
+cd src
+
+# Install uv if not already installed
+brew install uv || pip install uv
+
+# Install dependencies
+uv pip install -e .[test]
+
+# Run tests
+uv run pytest
+
+# Build package
+uv build
+```

{deltafi-2.30.0 → deltafi-2.30.1}/deltafi/result.py

@@ -287,6 +287,7 @@ class ChildTransformResult(TransformResult):
 
     def json(self):
         j = super().json()
+        j['messages'] = [message.json() for message in self.messages]
         if self.delta_file_name is not None:
             j['name'] = self.delta_file_name
         return j

deltafi-2.30.1/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "deltafi"
+version = "2.30.1"
+description = "SDK for DeltaFi plugins and actions"
+authors = [
+    {name = "DeltaFi", email = "deltafi@systolic.com"}
+]
+license = {text = "Apache License, Version 2.0"}
+readme = "README.md"
+keywords = ["deltafi"]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Software Development",
+    "Operating System :: OS Independent",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13"
+]
+requires-python = ">=3.13"
+dependencies = [
+    "deepdiff==6.7.1",
+    "json-logging==1.5.1",
+    "minio==7.2.15",
+    "PyYAML==6.0.2",
+    "pydantic==2.11.7",
+    "redis==6.2.0",
+    "requests==2.32.4",
+    "urllib3==2.5.0",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest==8.4.1",
+    "pytest-mock==3.14.1",
+    "mockito==1.5.4",
+]
+dev = [
+    "ruff>=0.1.0",
+    "mypy>=1.8.0",
+]
+
+[project.urls]
+"Source Code" = "https://gitlab.com/deltafi/deltafi"
+Documentation = "https://docs.deltafi.org/#/"
+"Bug Reports" = "https://chat.deltafi.org/deltafi/channels/bug-reports"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

deltafi-2.30.1/pyproject.toml.template

@@ -0,0 +1,50 @@
+[project]
+name = "deltafi"
+version = "0.0.0"
+description = "SDK for DeltaFi plugins and actions"
+authors = [
+    {name = "DeltaFi", email = "deltafi@systolic.com"}
+]
+license = {text = "Apache License, Version 2.0"}
+readme = "README.md"
+keywords = ["deltafi"]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Software Development",
+    "Operating System :: OS Independent",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13"
+]
+requires-python = ">=3.13"
+dependencies = [
+    "deepdiff==6.7.1",
+    "json-logging==1.5.1",
+    "minio==7.2.15",
+    "PyYAML==6.0.2",
+    "pydantic==2.11.7",
+    "redis==6.2.0",
+    "requests==2.32.4",
+    "urllib3==2.5.0",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest==8.4.1",
+    "pytest-mock==3.14.1",
+    "mockito==1.5.4",
+]
+dev = [
+    "ruff>=0.1.0",
+    "mypy>=1.8.0",
+]
+
+[project.urls]
+"Source Code" = "https://gitlab.com/deltafi/deltafi"
+Documentation = "https://docs.deltafi.org/#/"
+"Bug Reports" = "https://chat.deltafi.org/deltafi/channels/bug-reports"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

deltafi-2.30.1/test/__init__.py

@@ -0,0 +1,30 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import os
+import sys
+
+PROJECT_PATH = os.getcwd()
+SOURCE_PATH = os.path.join(
+    PROJECT_PATH, "src"
+)
+sys.path.append(SOURCE_PATH)
+TEST_PATH = os.path.join(
+    PROJECT_PATH, "src/test"
+)
+sys.path.append(TEST_PATH)

deltafi-2.30.1/test/helperutils.py

@@ -0,0 +1,81 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.domain import Context, Event
+from deltafi.storage import Segment
+
+SEG_ID = "1"
+TEST_DID = "123did"
+
+
+def make_segment(seg_id):
+    return Segment(uuid=seg_id, offset=0, size=100, did=TEST_DID)
+
+
+def make_context_dict():
+    return {
+        'did': TEST_DID,
+        'deltaFileName': 'FILENAME',
+        'dataSource': 'DATA_SOURCE',
+        'flowName': 'FLOW_NAME',
+        'flowId': 'FLOW_ID',
+        'actionName': 'ACTION_NAME',
+        'actionVersion': '1.0',
+        'hostname': 'HOSTNAME',
+        'systemName': 'SYSTEM_NAME'
+    }
+
+
+def make_content_dict(name):
+    return {
+        'name': name,
+        'segments': [make_segment(SEG_ID).json()],
+        'mediaType': 'xml'
+    }
+
+
+def make_delta_file_message_dict():
+    return {
+        'metadata': {'plKey1': 'valueA', 'plKey2': 'valueB'},
+        'contentList': [make_content_dict('CONTENT_NAME'), make_content_dict('CONTENT 2')]
+    }
+
+
+def make_context():
+    return Context(did=TEST_DID,
+                   delta_file_name='FILENAME',
+                   data_source='DATA_SOURCE',
+                   flow_name='FLOW_NAME',
+                   flow_id='FLOW_ID',
+                   action_name='ACTION_NAME',
+                   action_version='1.0',
+                   hostname='HOSTNAME',
+                   system_name='SYSTEM_NAME',
+                   content_service=None)
+
+
+def make_event(content_service):
+    logger = None
+    event = Event.create({
+        'deltaFileMessages': [make_delta_file_message_dict()],
+        'actionContext': make_context_dict(),
+        'actionParams': {
+            "thing": "theThing"
+        }
+    }, content_service, logger)
+    return event

deltafi-2.30.1/test/plugin_data/flows/variables.yaml

@@ -0,0 +1,6 @@
+- name: name1
+  description: d1
+- name: name2
+  description: d2
+- name: name3
+  description: d3

deltafi-2.30.1/test/plugin_data/flows/variables.yml

@@ -0,0 +1,8 @@
+name: name1
+description: d1
+---
+name: name2
+description: d2
+---
+name: name3
+description: d3

deltafi-2.30.1/test/plugin_data/tests/f1.yaml

@@ -0,0 +1,2 @@
+name: name1
+description: d1

deltafi-2.30.1/test/plugin_data/tests/f2.yml

@@ -0,0 +1,5 @@
+name: name2
+description: d2
+---
+name: name3
+description: d3

deltafi-2.30.1/test/plugin_data/tests/f3.txt

@@ -0,0 +1 @@
+This file will be ignored by _read_valid_files() due to its extension

deltafi-2.30.1/test/plugin_data/tests/f4.json

@@ -0,0 +1,4 @@
+{
+	"name": "name4",
+	"description": "d4"
+}

deltafi-2.30.1/test/sample/__init__.py

@@ -0,0 +1,19 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+

deltafi-2.30.1/test/sample/actions.py

@@ -0,0 +1,76 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from abc import abstractmethod
+from typing import Dict, List
+
+from deltafi.action import ActionOptions, DescriptionWithConditions, InputSpec, OutputSpec, TransformAction
+from deltafi.domain import Context, Content
+from deltafi.input import TransformInput
+from deltafi.result import TransformResult
+from deltafi.storage import ContentService
+from mockito import mock
+from pydantic import BaseModel, Field
+
+
+class SampleTransformParameters(BaseModel):
+    a_string: str = Field(description="this string parameter is required")
+    def_string: str = Field(default="default-val", description="str with default")
+    a_dict: dict[str, str] = Field(description="this dict parameter is required")
+    def_dict: Dict[str, str] = Field(default={"key1:" "val1"}, description="dict has default")
+    a_list: List[str] = Field(default=[], description="list with default")
+    a_bool: bool = Field(description="this boolean parameter is required")
+    def_int: int = Field(default=100, description="int with default")
+
+
+class SampleTransformAction(TransformAction):
+    def __init__(self):
+        super().__init__('', ActionOptions(description='Transform action description',
+                                           input_spec=InputSpec(content_summary='The input content summary', metadata_summary='The input metadata summary'),
+                                           output_spec=OutputSpec(content_summary='The output content summary', metadata_summary='The output metadata summary', annotations_summary='The output annotations summary'),
+                                           filters=[DescriptionWithConditions('Filter 1', ['Condition A', 'Condition B']),
+                                                   'Filter 2'],
+                                           errors=[DescriptionWithConditions('Error 1', ['Condition A', 'Condition B']),
+                                                   'Error 2'],
+                                           notes=['Note 1', 'Note2'],
+                                           details='The details'))
+
+    def param_class(self):
+        return SampleTransformParameters
+
+    def transform(self, context: Context, params: SampleTransformParameters, transform_input: TransformInput):
+        return TransformResult(context).add_metadata('transformKey', 'transformValue') \
+            .add_content(Content(name='transformed content', segments=[], media_type='text/plain',
+                                 content_service=mock(ContentService)))
+
+
+class SampleAbstractTransformAction(TransformAction):
+    def __init__(self):
+        super().__init__('Transform action description - ignored due to the abstract method')
+
+    def param_class(self):
+        return SampleTransformParameters
+
+    @abstractmethod
+    def extra_abstract_method(self):
+        pass
+
+    def transform(self, context: Context, params: SampleTransformParameters, transform_input: TransformInput):
+        return TransformResult(context).add_metadata('transformKey', 'transformValue') \
+            .add_content(Content(name='transformed content', segments=[], media_type='text/plain',
+                                 content_service=mock(ContentService)))

deltafi-2.30.1/test/test_actioneventqueue.py

@@ -0,0 +1,69 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import time
+
+import redis
+from deltafi.actioneventqueue import ActionEventQueue
+from mockito import when, mock, unstub
+
+TEST_URL = "http://127.0.0.1:12345"
+TEST_ACTION = "Action"
+TEST_ITEM = "1234"
+
+
+def test_action_event_queue_size():
+    unstub()
+    mock_pool = mock(redis.ConnectionPool)
+    mock_conn = mock(redis.Redis)
+
+    when(redis).ConnectionPool(...).thenReturn(mock_pool)
+    when(redis).Redis(connection_pool=mock_pool).thenReturn(mock_conn)
+    when(mock_conn).zcard(TEST_ACTION).thenReturn(1)
+
+    service = ActionEventQueue(TEST_URL, 5, "password", None)
+    assert service.size(TEST_ACTION) == 1
+
+
+def test_action_event_queue_put():
+    unstub()
+    mock_pool = mock(redis.ConnectionPool)
+    mock_conn = mock(redis.Redis)
+
+    when(redis).ConnectionPool(...).thenReturn(mock_pool)
+    when(redis).Redis(connection_pool=mock_pool).thenReturn(mock_conn)
+    when(time).time().thenReturn(1.6)
+    when(mock_conn).zadd(TEST_ACTION, {TEST_ITEM: 1600}, nx=True).thenReturn(1)
+
+    service = ActionEventQueue(TEST_URL, 5, "password", None)
+    assert service.put(TEST_ACTION, TEST_ITEM) == 1
+
+
+def test_action_event_queue_take():
+    unstub()
+    mock_pool = mock(redis.ConnectionPool)
+    mock_conn = mock(redis.Redis)
+
+    result_tuple = (TEST_ACTION, TEST_ITEM, 1600)
+
+    when(redis).ConnectionPool(...).thenReturn(mock_pool)
+    when(redis).Redis(connection_pool=mock_pool).thenReturn(mock_conn)
+    when(mock_conn).bzpopmin(TEST_ACTION, 0).thenReturn(result_tuple)
+
+    service = ActionEventQueue(TEST_URL, 5, "password", "some-pod")
+    assert service.take(TEST_ACTION) == TEST_ITEM