port-ocean 0.30.4__py3-none-any.whl → 0.30.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- port_ocean/core/integrations/mixins/utils.py +114 -12
- port_ocean/tests/core/integrations/mixins/test_integration_utils.py +494 -0
- {port_ocean-0.30.4.dist-info → port_ocean-0.30.5.dist-info}/METADATA +1 -1
- {port_ocean-0.30.4.dist-info → port_ocean-0.30.5.dist-info}/RECORD +7 -6
- {port_ocean-0.30.4.dist-info → port_ocean-0.30.5.dist-info}/LICENSE.md +0 -0
- {port_ocean-0.30.4.dist-info → port_ocean-0.30.5.dist-info}/WHEEL +0 -0
- {port_ocean-0.30.4.dist-info → port_ocean-0.30.5.dist-info}/entry_points.txt +0 -0
|
@@ -9,6 +9,7 @@ import subprocess
|
|
|
9
9
|
import tempfile
|
|
10
10
|
from contextlib import contextmanager
|
|
11
11
|
from typing import Any, AsyncGenerator, Awaitable, Callable, Generator, cast
|
|
12
|
+
import copy
|
|
12
13
|
|
|
13
14
|
import ijson
|
|
14
15
|
from loguru import logger
|
|
@@ -124,8 +125,8 @@ async def resync_generator_wrapper(
|
|
|
124
125
|
if items_to_parse:
|
|
125
126
|
for data in result:
|
|
126
127
|
data_path: str | None = None
|
|
127
|
-
if isinstance(data, dict) and data.get("
|
|
128
|
-
content = data
|
|
128
|
+
if isinstance(data, dict) and data.get("__type") == "path":
|
|
129
|
+
content = data.get("file", {}).get("content") if isinstance(data["file"].get("content"), dict) else {}
|
|
129
130
|
data_path = content.get("path", None)
|
|
130
131
|
bulks = get_items_to_parse_bulks(data, data_path, items_to_parse, items_to_parse_name, data.get("__base_jq", ".file.content"))
|
|
131
132
|
async for bulk in bulks:
|
|
@@ -147,6 +148,62 @@ async def resync_generator_wrapper(
|
|
|
147
148
|
"At least one of the resync generator iterations failed", errors
|
|
148
149
|
)
|
|
149
150
|
|
|
151
|
+
def extract_jq_deletion_path_revised(jq_expression: str) -> str | None:
|
|
152
|
+
"""
|
|
153
|
+
Revised function to extract a simple path suitable for del() by analyzing pipe segments.
|
|
154
|
+
"""
|
|
155
|
+
expr = jq_expression.strip()
|
|
156
|
+
|
|
157
|
+
# 1. Handle surrounding parentheses and extract the main chain
|
|
158
|
+
if expr.startswith('('):
|
|
159
|
+
match_paren = re.match(r'\((.*?)\)', expr, re.DOTALL)
|
|
160
|
+
if match_paren:
|
|
161
|
+
chain = match_paren.group(1).strip()
|
|
162
|
+
else:
|
|
163
|
+
return None
|
|
164
|
+
else:
|
|
165
|
+
chain = expr
|
|
166
|
+
|
|
167
|
+
# 2. Split the chain by the main pipe operator (excluding pipes inside quotes or brackets,
|
|
168
|
+
# but for simplicity here, we split naively and check segments)
|
|
169
|
+
segments = chain.split('|')
|
|
170
|
+
|
|
171
|
+
# 3. Analyze each segment for a simple path
|
|
172
|
+
for segment in segments:
|
|
173
|
+
segment = segment.strip()
|
|
174
|
+
|
|
175
|
+
# Ignore variable assignment segments like '. as $root'
|
|
176
|
+
if re.match(r'^\.\s+as\s+\$\w+', segment):
|
|
177
|
+
continue
|
|
178
|
+
|
|
179
|
+
# Ignore identity and variable access like '.' or '$items'
|
|
180
|
+
if segment == '.' or segment.startswith('$'):
|
|
181
|
+
continue
|
|
182
|
+
|
|
183
|
+
# Look for the first genuine path accessor (e.g., .key, .[index], .key.nested, .key[0])
|
|
184
|
+
# This regex looks for a starting dot and follows it with path components
|
|
185
|
+
# (alphanumeric keys or bracketed accessors, where brackets can follow words directly)
|
|
186
|
+
# Pattern: Start with .word or .[index], then optionally more:
|
|
187
|
+
# - .word (dot followed by word)
|
|
188
|
+
# - [index] (bracket directly after word, no dot)
|
|
189
|
+
# - .[index] (dot followed by bracket)
|
|
190
|
+
path_match = re.match(r'(\.[\w]+|\.\[[^\]]+\])(\.[\w]+|\[[^\]]+\]|\.\[[^\]]+\])*', segment)
|
|
191
|
+
|
|
192
|
+
if path_match:
|
|
193
|
+
path = path_match.group(0).strip()
|
|
194
|
+
|
|
195
|
+
# If the path is immediately followed by a simple fallback (// value),
|
|
196
|
+
# we consider the path complete.
|
|
197
|
+
if re.search(r'\s*//\s*(\[\]|null|\.|\{.*?\})', segment):
|
|
198
|
+
return path
|
|
199
|
+
|
|
200
|
+
# If the path is just a path segment followed by nothing or the end of a complex
|
|
201
|
+
# expression (like .file.content.raw) we return it.
|
|
202
|
+
return path
|
|
203
|
+
|
|
204
|
+
# Default case: No suitable path found after checking all segments
|
|
205
|
+
return None
|
|
206
|
+
|
|
150
207
|
|
|
151
208
|
def is_resource_supported(
|
|
152
209
|
kind: str, resync_event_mapping: dict[str | None, list[RESYNC_EVENT_LISTENER]]
|
|
@@ -197,6 +254,7 @@ async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, ite
|
|
|
197
254
|
temp_output_path = None
|
|
198
255
|
|
|
199
256
|
try:
|
|
257
|
+
is_path_type = True
|
|
200
258
|
# Create secure temporary files
|
|
201
259
|
if not data_path:
|
|
202
260
|
raw_data_serialized = json.dumps(raw_data)
|
|
@@ -204,17 +262,17 @@ async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, ite
|
|
|
204
262
|
with open(temp_data_path, "w") as f:
|
|
205
263
|
f.write(raw_data_serialized)
|
|
206
264
|
data_path = temp_data_path
|
|
265
|
+
is_path_type = False
|
|
207
266
|
|
|
208
267
|
temp_output_path = _create_secure_temp_file("_parsed.json")
|
|
209
268
|
|
|
210
|
-
delete_target =
|
|
211
|
-
base_jq_object_string = await _build_base_jq_object_string(raw_data, base_jq, delete_target)
|
|
269
|
+
delete_target = extract_jq_deletion_path_revised(items_to_parse) or '.'
|
|
212
270
|
|
|
213
271
|
# Build jq expression safely
|
|
214
272
|
jq_expression = f""". as $all
|
|
215
273
|
| ($all | {items_to_parse}) as $items
|
|
216
274
|
| $items
|
|
217
|
-
|
|
|
275
|
+
| {_build_mapping_jq_expression(items_to_parse_name, base_jq, delete_target, is_path_type)}"""
|
|
218
276
|
|
|
219
277
|
# Use subprocess with list arguments instead of shell=True
|
|
220
278
|
|
|
@@ -237,7 +295,7 @@ async def get_items_to_parse_bulks(raw_data: dict[Any, Any], data_path: str, ite
|
|
|
237
295
|
with open(temp_output_path, "r") as f:
|
|
238
296
|
events_stream = get_events_as_a_stream(f, 'item', ocean.config.yield_items_to_parse_batch_size)
|
|
239
297
|
for items_bulk in events_stream:
|
|
240
|
-
yield items_bulk
|
|
298
|
+
yield items_bulk if not is_path_type else [merge_raw_data_to_item(item, raw_data) for item in items_bulk]
|
|
241
299
|
|
|
242
300
|
except ValueError as e:
|
|
243
301
|
logger.error(f"Invalid jq expression: {e}")
|
|
@@ -260,13 +318,57 @@ def unsupported_kind_response(
|
|
|
260
318
|
logger.error(f"Kind {kind} is not supported in this integration")
|
|
261
319
|
return [], [KindNotImplementedException(kind, available_resync_kinds)]
|
|
262
320
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
321
|
+
def _build_mapping_jq_expression(items_to_parse_name: str, base_jq: str, delete_target: str, is_path_type: bool = False) -> str:
|
|
322
|
+
if is_path_type:
|
|
323
|
+
return f"map({{{items_to_parse_name}: . }} | {base_jq} = (($all | del({delete_target})) // {{}}))"
|
|
324
|
+
return f"map(($all | del({delete_target})) + {{{items_to_parse_name}: . }})"
|
|
325
|
+
|
|
326
|
+
def merge_raw_data_to_item(item: dict[str, Any], raw_data: dict[str, Any]) -> dict[str, Any]:
|
|
327
|
+
return recursive_dict_merge(raw_data, item)
|
|
328
|
+
|
|
329
|
+
def recursive_dict_merge(d1: dict[Any, Any], d2: dict[Any, Any]) -> dict[Any, Any]:
|
|
330
|
+
"""
|
|
331
|
+
Recursively merges dict d2 into dict d1.
|
|
332
|
+
|
|
333
|
+
If a key exists in both dictionaries:
|
|
334
|
+
1. If the value in d2 is an empty dictionary (e.g., {}), it overwrites the value in d1.
|
|
335
|
+
2. If both values are non-empty dictionaries, they are merged recursively.
|
|
336
|
+
3. Otherwise, the value from d2 overwrites the value from d1.
|
|
337
|
+
|
|
338
|
+
The original dictionaries are not modified (d1 is copied).
|
|
339
|
+
|
|
340
|
+
Args:
|
|
341
|
+
d1: The base dictionary (will be copied and modified).
|
|
342
|
+
d2: The dictionary to merge into d1.
|
|
343
|
+
|
|
344
|
+
Returns:
|
|
345
|
+
The merged dictionary.
|
|
346
|
+
"""
|
|
347
|
+
# Start with a copy of d1 to ensure d1 is not mutated
|
|
348
|
+
merged_dict = copy.deepcopy(d1)
|
|
349
|
+
|
|
350
|
+
for key, value in d2.items():
|
|
351
|
+
# Condition to trigger recursive deep merge:
|
|
352
|
+
# 1. Key exists in merged_dict
|
|
353
|
+
# 2. Both values are dictionaries
|
|
354
|
+
# 3. The value from d2 is NOT an empty dictionary ({}).
|
|
355
|
+
# If d2's value is {}, we treat it as an explicit instruction to overwrite/clear.
|
|
356
|
+
is_deep_merge = (
|
|
357
|
+
key in merged_dict and
|
|
358
|
+
isinstance(merged_dict[key], dict) and
|
|
359
|
+
isinstance(value, dict) and
|
|
360
|
+
value != {}
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
if is_deep_merge:
|
|
364
|
+
# If both are dictionaries and d2 is not empty, recurse
|
|
365
|
+
merged_dict[key] = recursive_dict_merge(merged_dict[key], value)
|
|
366
|
+
else:
|
|
367
|
+
# Otherwise (new key, non-dict value, or explicit {} overwrite),
|
|
368
|
+
# overwrite the value from d1 with the value from d2
|
|
369
|
+
merged_dict[key] = value
|
|
269
370
|
|
|
371
|
+
return merged_dict
|
|
270
372
|
|
|
271
373
|
def get_events_as_a_stream(
|
|
272
374
|
stream: Any,
|
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from port_ocean.core.integrations.mixins.utils import (
|
|
4
|
+
_build_mapping_jq_expression,
|
|
5
|
+
extract_jq_deletion_path_revised,
|
|
6
|
+
recursive_dict_merge,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestBuildMappingJqExpression:
|
|
11
|
+
"""Tests for _build_mapping_jq_expression function."""
|
|
12
|
+
|
|
13
|
+
def test_build_mapping_jq_expression_non_path_type(self) -> None:
|
|
14
|
+
"""Test jq expression building for non-path type."""
|
|
15
|
+
items_to_parse_name = "items"
|
|
16
|
+
base_jq = ".file.content"
|
|
17
|
+
delete_target = ".file.content.raw"
|
|
18
|
+
|
|
19
|
+
result = _build_mapping_jq_expression(
|
|
20
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=False
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
expected = "map(($all | del(.file.content.raw)) + {items: . })"
|
|
24
|
+
assert result == expected
|
|
25
|
+
|
|
26
|
+
def test_build_mapping_jq_expression_path_type(self) -> None:
|
|
27
|
+
"""Test jq expression building for path type."""
|
|
28
|
+
items_to_parse_name = "items"
|
|
29
|
+
base_jq = ".file.content"
|
|
30
|
+
delete_target = ".file.content.raw"
|
|
31
|
+
|
|
32
|
+
result = _build_mapping_jq_expression(
|
|
33
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=True
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
expected = "map({items: . } | .file.content = (($all | del(.file.content.raw)) // {}))"
|
|
37
|
+
assert result == expected
|
|
38
|
+
|
|
39
|
+
def test_build_mapping_jq_expression_with_different_delete_target(self) -> None:
|
|
40
|
+
"""Test jq expression building with different delete targets."""
|
|
41
|
+
items_to_parse_name = "parsed_data"
|
|
42
|
+
base_jq = ".data"
|
|
43
|
+
delete_target = ".data.temp"
|
|
44
|
+
|
|
45
|
+
# Non-path type
|
|
46
|
+
result_non_path = _build_mapping_jq_expression(
|
|
47
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=False
|
|
48
|
+
)
|
|
49
|
+
expected_non_path = "map(($all | del(.data.temp)) + {parsed_data: . })"
|
|
50
|
+
assert result_non_path == expected_non_path
|
|
51
|
+
|
|
52
|
+
# Path type
|
|
53
|
+
result_path = _build_mapping_jq_expression(
|
|
54
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=True
|
|
55
|
+
)
|
|
56
|
+
expected_path = "map({parsed_data: . } | .data = (($all | del(.data.temp)) // {}))"
|
|
57
|
+
assert result_path == expected_path
|
|
58
|
+
|
|
59
|
+
def test_build_mapping_jq_expression_with_simple_delete_target(self) -> None:
|
|
60
|
+
"""Test jq expression building with simple delete target."""
|
|
61
|
+
items_to_parse_name = "items"
|
|
62
|
+
base_jq = "."
|
|
63
|
+
delete_target = "."
|
|
64
|
+
|
|
65
|
+
# Non-path type
|
|
66
|
+
result_non_path = _build_mapping_jq_expression(
|
|
67
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=False
|
|
68
|
+
)
|
|
69
|
+
expected_non_path = "map(($all | del(.)) + {items: . })"
|
|
70
|
+
assert result_non_path == expected_non_path
|
|
71
|
+
|
|
72
|
+
# Path type
|
|
73
|
+
result_path = _build_mapping_jq_expression(
|
|
74
|
+
items_to_parse_name, base_jq, delete_target, is_path_type=True
|
|
75
|
+
)
|
|
76
|
+
expected_path = "map({items: . } | . = (($all | del(.)) // {}))"
|
|
77
|
+
assert result_path == expected_path
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class TestRecursiveDictMerge:
|
|
81
|
+
"""Tests for recursive_dict_merge function."""
|
|
82
|
+
|
|
83
|
+
def test_simple_merge_new_keys(self) -> None:
|
|
84
|
+
"""Test merging dictionaries with new keys."""
|
|
85
|
+
d1 = {"a": 1, "b": 2}
|
|
86
|
+
d2 = {"c": 3, "d": 4}
|
|
87
|
+
|
|
88
|
+
result = recursive_dict_merge(d1, d2)
|
|
89
|
+
|
|
90
|
+
assert result == {"a": 1, "b": 2, "c": 3, "d": 4}
|
|
91
|
+
# Ensure original dicts are not modified
|
|
92
|
+
assert d1 == {"a": 1, "b": 2}
|
|
93
|
+
assert d2 == {"c": 3, "d": 4}
|
|
94
|
+
|
|
95
|
+
def test_simple_overwrite(self) -> None:
|
|
96
|
+
"""Test overwriting values with non-dict values."""
|
|
97
|
+
d1 = {"a": 1, "b": 2}
|
|
98
|
+
d2 = {"b": 3, "c": 4}
|
|
99
|
+
|
|
100
|
+
result = recursive_dict_merge(d1, d2)
|
|
101
|
+
|
|
102
|
+
assert result == {"a": 1, "b": 3, "c": 4}
|
|
103
|
+
|
|
104
|
+
def test_deep_merge(self) -> None:
|
|
105
|
+
"""Test recursive merging of nested dictionaries."""
|
|
106
|
+
d1 = {"a": 1, "nested": {"x": 10, "y": 20}}
|
|
107
|
+
d2 = {"b": 2, "nested": {"y": 30, "z": 40}}
|
|
108
|
+
|
|
109
|
+
result = recursive_dict_merge(d1, d2)
|
|
110
|
+
|
|
111
|
+
assert result == {"a": 1, "b": 2, "nested": {"x": 10, "y": 30, "z": 40}}
|
|
112
|
+
|
|
113
|
+
def test_empty_dict_overwrite(self) -> None:
|
|
114
|
+
"""Test that empty dict in d2 overwrites non-empty dict in d1."""
|
|
115
|
+
d1 = {"a": 1, "nested": {"x": 10, "y": 20}}
|
|
116
|
+
d2 = {"nested": {}}
|
|
117
|
+
|
|
118
|
+
result = recursive_dict_merge(d1, d2)
|
|
119
|
+
|
|
120
|
+
assert result == {"a": 1, "nested": {}}
|
|
121
|
+
|
|
122
|
+
def test_empty_dict_overwrite_multiple_levels(self) -> None:
|
|
123
|
+
"""Test empty dict overwrite at multiple nesting levels."""
|
|
124
|
+
d1 = {
|
|
125
|
+
"level1": {
|
|
126
|
+
"level2": {
|
|
127
|
+
"level3": {"value": 100}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
d2 = {
|
|
132
|
+
"level1": {
|
|
133
|
+
"level2": {}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
result = recursive_dict_merge(d1, d2)
|
|
138
|
+
|
|
139
|
+
assert result == {"level1": {"level2": {}}}
|
|
140
|
+
|
|
141
|
+
def test_mixed_types(self) -> None:
|
|
142
|
+
"""Test merging with mixed value types."""
|
|
143
|
+
d1 = {
|
|
144
|
+
"string": "hello",
|
|
145
|
+
"number": 42,
|
|
146
|
+
"list": [1, 2, 3],
|
|
147
|
+
"nested": {"a": 1}
|
|
148
|
+
}
|
|
149
|
+
d2 = {
|
|
150
|
+
"string": "world",
|
|
151
|
+
"number": 100,
|
|
152
|
+
"list": [4, 5, 6],
|
|
153
|
+
"nested": {"b": 2}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
result = recursive_dict_merge(d1, d2)
|
|
157
|
+
|
|
158
|
+
assert result == {
|
|
159
|
+
"string": "world",
|
|
160
|
+
"number": 100,
|
|
161
|
+
"list": [4, 5, 6],
|
|
162
|
+
"nested": {"a": 1, "b": 2}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
def test_deep_nested_merge(self) -> None:
|
|
166
|
+
"""Test merging deeply nested structures."""
|
|
167
|
+
d1 = {
|
|
168
|
+
"a": {
|
|
169
|
+
"b": {
|
|
170
|
+
"c": {
|
|
171
|
+
"d": 1,
|
|
172
|
+
"e": 2
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
d2 = {
|
|
178
|
+
"a": {
|
|
179
|
+
"b": {
|
|
180
|
+
"c": {
|
|
181
|
+
"e": 3,
|
|
182
|
+
"f": 4
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
result = recursive_dict_merge(d1, d2)
|
|
189
|
+
|
|
190
|
+
assert result == {
|
|
191
|
+
"a": {
|
|
192
|
+
"b": {
|
|
193
|
+
"c": {
|
|
194
|
+
"d": 1,
|
|
195
|
+
"e": 3,
|
|
196
|
+
"f": 4
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
def test_empty_dictionaries(self) -> None:
|
|
203
|
+
"""Test merging with empty dictionaries."""
|
|
204
|
+
d1 = {}
|
|
205
|
+
d2 = {"a": 1}
|
|
206
|
+
|
|
207
|
+
result = recursive_dict_merge(d1, d2)
|
|
208
|
+
|
|
209
|
+
assert result == {"a": 1}
|
|
210
|
+
|
|
211
|
+
def test_both_empty_dictionaries(self) -> None:
|
|
212
|
+
"""Test merging two empty dictionaries."""
|
|
213
|
+
d1 = {}
|
|
214
|
+
d2 = {}
|
|
215
|
+
|
|
216
|
+
result = recursive_dict_merge(d1, d2)
|
|
217
|
+
|
|
218
|
+
assert result == {}
|
|
219
|
+
|
|
220
|
+
def test_immutability_d1(self) -> None:
|
|
221
|
+
"""Test that d1 is not modified by the merge operation."""
|
|
222
|
+
d1 = {"a": 1, "nested": {"x": 10}}
|
|
223
|
+
d2 = {"b": 2, "nested": {"y": 20}}
|
|
224
|
+
|
|
225
|
+
original_d1 = {"a": 1, "nested": {"x": 10}}
|
|
226
|
+
result = recursive_dict_merge(d1, d2)
|
|
227
|
+
|
|
228
|
+
# Verify d1 is unchanged
|
|
229
|
+
assert d1 == original_d1
|
|
230
|
+
# Verify result is correct
|
|
231
|
+
assert result == {"a": 1, "b": 2, "nested": {"x": 10, "y": 20}}
|
|
232
|
+
|
|
233
|
+
def test_immutability_d2(self) -> None:
|
|
234
|
+
"""Test that d2 is not modified by the merge operation."""
|
|
235
|
+
d1 = {"a": 1}
|
|
236
|
+
d2 = {"b": 2, "nested": {"x": 10}}
|
|
237
|
+
|
|
238
|
+
original_d2 = {"b": 2, "nested": {"x": 10}}
|
|
239
|
+
result = recursive_dict_merge(d1, d2)
|
|
240
|
+
|
|
241
|
+
# Verify d2 is unchanged
|
|
242
|
+
assert d2 == original_d2
|
|
243
|
+
# Verify result is correct
|
|
244
|
+
assert result == {"a": 1, "b": 2, "nested": {"x": 10}}
|
|
245
|
+
|
|
246
|
+
def test_non_dict_value_overwrites_dict(self) -> None:
|
|
247
|
+
"""Test that non-dict value in d2 overwrites dict value in d1."""
|
|
248
|
+
d1 = {"key": {"nested": "value"}}
|
|
249
|
+
d2 = {"key": "simple_string"}
|
|
250
|
+
|
|
251
|
+
result = recursive_dict_merge(d1, d2)
|
|
252
|
+
|
|
253
|
+
assert result == {"key": "simple_string"}
|
|
254
|
+
|
|
255
|
+
def test_dict_overwrites_non_dict_value(self) -> None:
|
|
256
|
+
"""Test that dict value in d2 overwrites non-dict value in d1."""
|
|
257
|
+
d1 = {"key": "simple_string"}
|
|
258
|
+
d2 = {"key": {"nested": "value"}}
|
|
259
|
+
|
|
260
|
+
result = recursive_dict_merge(d1, d2)
|
|
261
|
+
|
|
262
|
+
assert result == {"key": {"nested": "value"}}
|
|
263
|
+
|
|
264
|
+
def test_complex_real_world_scenario(self) -> None:
|
|
265
|
+
"""Test a complex real-world merge scenario."""
|
|
266
|
+
d1 = {
|
|
267
|
+
"metadata": {
|
|
268
|
+
"version": "1.0",
|
|
269
|
+
"author": "Alice",
|
|
270
|
+
"tags": ["python", "testing"]
|
|
271
|
+
},
|
|
272
|
+
"data": {
|
|
273
|
+
"users": {
|
|
274
|
+
"count": 100,
|
|
275
|
+
"active": 80
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
"config": {
|
|
279
|
+
"debug": False
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
d2 = {
|
|
283
|
+
"metadata": {
|
|
284
|
+
"version": "2.0",
|
|
285
|
+
"tags": ["python", "testing", "advanced"]
|
|
286
|
+
},
|
|
287
|
+
"data": {
|
|
288
|
+
"users": {
|
|
289
|
+
"active": 90
|
|
290
|
+
},
|
|
291
|
+
"posts": {
|
|
292
|
+
"count": 200
|
|
293
|
+
}
|
|
294
|
+
},
|
|
295
|
+
"config": {}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
result = recursive_dict_merge(d1, d2)
|
|
299
|
+
|
|
300
|
+
assert result == {
|
|
301
|
+
"metadata": {
|
|
302
|
+
"version": "2.0",
|
|
303
|
+
"author": "Alice",
|
|
304
|
+
"tags": ["python", "testing", "advanced"]
|
|
305
|
+
},
|
|
306
|
+
"data": {
|
|
307
|
+
"users": {
|
|
308
|
+
"count": 100,
|
|
309
|
+
"active": 90
|
|
310
|
+
},
|
|
311
|
+
"posts": {
|
|
312
|
+
"count": 200
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
"config": {}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class TestExtractJqDeletionPathRevised:
|
|
320
|
+
"""Tests for extract_jq_deletion_path_revised function."""
|
|
321
|
+
|
|
322
|
+
def test_simple_path(self) -> None:
|
|
323
|
+
"""Test extraction of simple path like .key."""
|
|
324
|
+
result = extract_jq_deletion_path_revised(".file.content")
|
|
325
|
+
assert result == ".file.content"
|
|
326
|
+
|
|
327
|
+
def test_simple_single_key(self) -> None:
|
|
328
|
+
"""Test extraction of single key path."""
|
|
329
|
+
result = extract_jq_deletion_path_revised(".key")
|
|
330
|
+
assert result == ".key"
|
|
331
|
+
|
|
332
|
+
def test_nested_path(self) -> None:
|
|
333
|
+
"""Test extraction of nested path."""
|
|
334
|
+
result = extract_jq_deletion_path_revised(".file.content.raw")
|
|
335
|
+
assert result == ".file.content.raw"
|
|
336
|
+
|
|
337
|
+
def test_path_with_parentheses(self) -> None:
|
|
338
|
+
"""Test extraction with surrounding parentheses."""
|
|
339
|
+
result = extract_jq_deletion_path_revised("(.file.content)")
|
|
340
|
+
assert result == ".file.content"
|
|
341
|
+
|
|
342
|
+
def test_path_with_pipe_segments(self) -> None:
|
|
343
|
+
"""Test extraction from pipe-separated expression."""
|
|
344
|
+
result = extract_jq_deletion_path_revised(". as $all | .file.content")
|
|
345
|
+
assert result == ".file.content"
|
|
346
|
+
|
|
347
|
+
def test_path_with_variable_assignment_ignored(self) -> None:
|
|
348
|
+
"""Test that variable assignment segments are ignored."""
|
|
349
|
+
result = extract_jq_deletion_path_revised(". as $root | .file.content")
|
|
350
|
+
assert result == ".file.content"
|
|
351
|
+
|
|
352
|
+
def test_path_with_variable_access_ignored(self) -> None:
|
|
353
|
+
"""Test that variable access like $items is ignored."""
|
|
354
|
+
result = extract_jq_deletion_path_revised("$items | .file.content")
|
|
355
|
+
assert result == ".file.content"
|
|
356
|
+
|
|
357
|
+
def test_identity_operator_ignored(self) -> None:
|
|
358
|
+
"""Test that identity operator '.' is ignored."""
|
|
359
|
+
result = extract_jq_deletion_path_revised(". | .file.content")
|
|
360
|
+
assert result == ".file.content"
|
|
361
|
+
|
|
362
|
+
def test_path_with_fallback_operator(self) -> None:
|
|
363
|
+
"""Test path extraction with fallback operator (//)."""
|
|
364
|
+
result = extract_jq_deletion_path_revised(".file.content // {}")
|
|
365
|
+
assert result == ".file.content"
|
|
366
|
+
|
|
367
|
+
def test_path_with_fallback_to_null(self) -> None:
|
|
368
|
+
"""Test path extraction with fallback to null."""
|
|
369
|
+
result = extract_jq_deletion_path_revised(".file.content // null")
|
|
370
|
+
assert result == ".file.content"
|
|
371
|
+
|
|
372
|
+
def test_path_with_fallback_to_empty_array(self) -> None:
|
|
373
|
+
"""Test path extraction with fallback to empty array."""
|
|
374
|
+
result = extract_jq_deletion_path_revised(".file.content // []")
|
|
375
|
+
assert result == ".file.content"
|
|
376
|
+
|
|
377
|
+
def test_path_with_fallback_to_object(self) -> None:
|
|
378
|
+
"""Test path extraction with fallback to object."""
|
|
379
|
+
result = extract_jq_deletion_path_revised(".file.content // {}")
|
|
380
|
+
assert result == ".file.content"
|
|
381
|
+
|
|
382
|
+
def test_path_with_bracketed_accessor(self) -> None:
|
|
383
|
+
"""Test path extraction with array index accessor."""
|
|
384
|
+
result = extract_jq_deletion_path_revised(".[0].key")
|
|
385
|
+
assert result == ".[0].key"
|
|
386
|
+
|
|
387
|
+
def test_path_with_multiple_bracketed_accessors(self) -> None:
|
|
388
|
+
"""Test path extraction with multiple array index accessors."""
|
|
389
|
+
result = extract_jq_deletion_path_revised(".[0].[1].key")
|
|
390
|
+
assert result == ".[0].[1].key"
|
|
391
|
+
|
|
392
|
+
def test_path_with_mixed_accessors(self) -> None:
|
|
393
|
+
"""Test path extraction with mixed key and bracket accessors."""
|
|
394
|
+
result = extract_jq_deletion_path_revised(".file[0].content")
|
|
395
|
+
assert result == ".file[0].content"
|
|
396
|
+
|
|
397
|
+
def test_path_with_mixed_accessors_with_dots(self) -> None:
|
|
398
|
+
"""Test path extraction with mixed key and bracket accessors with dots."""
|
|
399
|
+
result = extract_jq_deletion_path_revised(".file.[0].content")
|
|
400
|
+
assert result == ".file.[0].content"
|
|
401
|
+
|
|
402
|
+
def test_complex_pipe_expression(self) -> None:
|
|
403
|
+
"""Test extraction from complex pipe expression."""
|
|
404
|
+
result = extract_jq_deletion_path_revised(
|
|
405
|
+
". as $all | ($all | .file.content) as $items | $items"
|
|
406
|
+
)
|
|
407
|
+
assert result == ".file.content"
|
|
408
|
+
|
|
409
|
+
def test_path_in_parentheses_with_pipes(self) -> None:
|
|
410
|
+
"""Test path extraction from parenthesized expression with pipes."""
|
|
411
|
+
result = extract_jq_deletion_path_revised("(. as $all | .file.content)")
|
|
412
|
+
assert result == ".file.content"
|
|
413
|
+
|
|
414
|
+
def test_no_path_returns_none(self) -> None:
|
|
415
|
+
"""Test that expression without path returns None."""
|
|
416
|
+
result = extract_jq_deletion_path_revised("$items")
|
|
417
|
+
assert result is None
|
|
418
|
+
|
|
419
|
+
def test_only_identity_returns_none(self) -> None:
|
|
420
|
+
"""Test that expression with only identity operator returns None."""
|
|
421
|
+
result = extract_jq_deletion_path_revised(".")
|
|
422
|
+
assert result is None
|
|
423
|
+
|
|
424
|
+
def test_only_variable_returns_none(self) -> None:
|
|
425
|
+
"""Test that expression with only variable returns None."""
|
|
426
|
+
result = extract_jq_deletion_path_revised("$items")
|
|
427
|
+
assert result is None
|
|
428
|
+
|
|
429
|
+
def test_only_variable_assignment_returns_none(self) -> None:
|
|
430
|
+
"""Test that expression with only variable assignment returns None."""
|
|
431
|
+
result = extract_jq_deletion_path_revised(". as $root")
|
|
432
|
+
assert result is None
|
|
433
|
+
|
|
434
|
+
def test_malformed_parentheses_returns_none(self) -> None:
|
|
435
|
+
"""Test that malformed parentheses return None."""
|
|
436
|
+
result = extract_jq_deletion_path_revised("(.file.content")
|
|
437
|
+
assert result is None
|
|
438
|
+
|
|
439
|
+
def test_whitespace_handling(self) -> None:
|
|
440
|
+
"""Test that whitespace is properly handled."""
|
|
441
|
+
result = extract_jq_deletion_path_revised(" .file.content ")
|
|
442
|
+
assert result == ".file.content"
|
|
443
|
+
|
|
444
|
+
def test_path_with_underscores(self) -> None:
|
|
445
|
+
"""Test path extraction with underscores in keys."""
|
|
446
|
+
result = extract_jq_deletion_path_revised(".file_content.raw_data")
|
|
447
|
+
assert result == ".file_content.raw_data"
|
|
448
|
+
|
|
449
|
+
def test_path_with_numbers_in_keys(self) -> None:
|
|
450
|
+
"""Test path extraction with numbers in keys."""
|
|
451
|
+
result = extract_jq_deletion_path_revised(".file2.content3")
|
|
452
|
+
assert result == ".file2.content3"
|
|
453
|
+
|
|
454
|
+
def test_deeply_nested_path(self) -> None:
|
|
455
|
+
"""Test extraction of deeply nested path."""
|
|
456
|
+
result = extract_jq_deletion_path_revised(".a.b.c.d.e.f")
|
|
457
|
+
assert result == ".a.b.c.d.e.f"
|
|
458
|
+
|
|
459
|
+
def test_path_with_fallback_in_pipe(self) -> None:
|
|
460
|
+
"""Test path extraction with fallback in pipe expression."""
|
|
461
|
+
result = extract_jq_deletion_path_revised(". as $all | .file.content // {}")
|
|
462
|
+
assert result == ".file.content"
|
|
463
|
+
|
|
464
|
+
def test_multiple_paths_returns_first(self) -> None:
|
|
465
|
+
"""Test that first valid path is returned when multiple exist."""
|
|
466
|
+
result = extract_jq_deletion_path_revised(".first.path | .second.path")
|
|
467
|
+
assert result == ".first.path"
|
|
468
|
+
|
|
469
|
+
def test_path_with_complex_bracket_expression(self) -> None:
|
|
470
|
+
"""Test path extraction with complex bracket expression."""
|
|
471
|
+
result = extract_jq_deletion_path_revised(".[\"key\"].value")
|
|
472
|
+
assert result == ".[\"key\"].value"
|
|
473
|
+
|
|
474
|
+
def test_empty_string_returns_none(self) -> None:
|
|
475
|
+
"""Test that empty string returns None."""
|
|
476
|
+
result = extract_jq_deletion_path_revised("")
|
|
477
|
+
assert result is None
|
|
478
|
+
|
|
479
|
+
def test_whitespace_only_returns_none(self) -> None:
|
|
480
|
+
"""Test that whitespace-only string returns None."""
|
|
481
|
+
result = extract_jq_deletion_path_revised(" ")
|
|
482
|
+
assert result is None
|
|
483
|
+
|
|
484
|
+
def test_real_world_file_content_path(self) -> None:
|
|
485
|
+
"""Test real-world scenario: file.content path."""
|
|
486
|
+
result = extract_jq_deletion_path_revised(".file.content.raw")
|
|
487
|
+
assert result == ".file.content.raw"
|
|
488
|
+
|
|
489
|
+
def test_real_world_with_variable_and_fallback(self) -> None:
|
|
490
|
+
"""Test real-world scenario with variable and fallback."""
|
|
491
|
+
result = extract_jq_deletion_path_revised(
|
|
492
|
+
". as $all | ($all | .file.content.raw) // {}"
|
|
493
|
+
)
|
|
494
|
+
assert result == ".file.content.raw"
|
|
@@ -130,7 +130,7 @@ port_ocean/core/integrations/mixins/handler.py,sha256=mZ7-0UlG3LcrwJttFbMe-R4xcO
|
|
|
130
130
|
port_ocean/core/integrations/mixins/live_events.py,sha256=zM24dhNc7uHx9XYZ6toVhDADPA90EnpOmZxgDegFZbA,4196
|
|
131
131
|
port_ocean/core/integrations/mixins/sync.py,sha256=Vm_898pLKBwfVewtwouDWsXoxcOLicnAy6pzyqqk6U8,4053
|
|
132
132
|
port_ocean/core/integrations/mixins/sync_raw.py,sha256=kcL7flnQ25E3KKyo6L3aL9wSzgBtoWYzgQjS4uRbDOs,42612
|
|
133
|
-
port_ocean/core/integrations/mixins/utils.py,sha256=
|
|
133
|
+
port_ocean/core/integrations/mixins/utils.py,sha256=nvLxzbRqOpbVhpI0xbKczrTCtLcR0HysFxck9b1BBy0,17425
|
|
134
134
|
port_ocean/core/models.py,sha256=8ZNEmM3Nq0VSB3fYJVgEdJVjJmaGjMnng-bm-ZBbTNg,3695
|
|
135
135
|
port_ocean/core/ocean_types.py,sha256=bkLlTd8XfJK6_JDl0eXUHfE_NygqgiInSMwJ4YJH01Q,1399
|
|
136
136
|
port_ocean/core/utils/entity_topological_sorter.py,sha256=MDUjM6OuDy4Xj68o-7InNN0w1jqjxeDfeY8U02vySNI,3081
|
|
@@ -191,6 +191,7 @@ port_ocean/tests/core/handlers/queue/test_local_queue.py,sha256=9Ly0HzZXbs6Rbl_b
|
|
|
191
191
|
port_ocean/tests/core/handlers/webhook/test_abstract_webhook_processor.py,sha256=zKwHhPAYEZoZ5Z2UETp1t--mbkS8uyvlXThB0obZTTc,3340
|
|
192
192
|
port_ocean/tests/core/handlers/webhook/test_processor_manager.py,sha256=wKzKO79HByqtLcKoYUQ6PjZ-VZAUT1TwdZXyH9NchfY,52365
|
|
193
193
|
port_ocean/tests/core/handlers/webhook/test_webhook_event.py,sha256=oR4dEHLO65mp6rkfNfszZcfFoRZlB8ZWee4XetmsuIk,3181
|
|
194
|
+
port_ocean/tests/core/integrations/mixins/test_integration_utils.py,sha256=MDNCnx5V_XVmUgeJB68Mwpm57_fbpWksijKe7aTMU4k,17552
|
|
194
195
|
port_ocean/tests/core/test_utils.py,sha256=Z3kdhb5V7Svhcyy3EansdTpgHL36TL6erNtU-OPwAcI,2647
|
|
195
196
|
port_ocean/tests/core/utils/test_entity_topological_sorter.py,sha256=zuq5WSPy_88PemG3mOUIHTxWMR_js1R7tOzUYlgBd68,3447
|
|
196
197
|
port_ocean/tests/core/utils/test_get_port_diff.py,sha256=YoQxAHZdX5nVpvrKV5Aox-jQ4w14AbfJVo-QK-ICAb8,4297
|
|
@@ -221,8 +222,8 @@ port_ocean/utils/repeat.py,sha256=U2OeCkHPWXmRTVoPV-VcJRlQhcYqPWI5NfmPlb1JIbc,32
|
|
|
221
222
|
port_ocean/utils/signal.py,sha256=J1sI-e_32VHP_VUa5bskLMFoJjJOAk5isrnewKDikUI,2125
|
|
222
223
|
port_ocean/utils/time.py,sha256=pufAOH5ZQI7gXvOvJoQXZXZJV-Dqktoj9Qp9eiRwmJ4,1939
|
|
223
224
|
port_ocean/version.py,sha256=UsuJdvdQlazzKGD3Hd5-U7N69STh8Dq9ggJzQFnu9fU,177
|
|
224
|
-
port_ocean-0.30.
|
|
225
|
-
port_ocean-0.30.
|
|
226
|
-
port_ocean-0.30.
|
|
227
|
-
port_ocean-0.30.
|
|
228
|
-
port_ocean-0.30.
|
|
225
|
+
port_ocean-0.30.5.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
226
|
+
port_ocean-0.30.5.dist-info/METADATA,sha256=bTE9t8Hx2DR2YUvA0RGM3AGwdvmYn3mCXTxJUJL66UU,7095
|
|
227
|
+
port_ocean-0.30.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
228
|
+
port_ocean-0.30.5.dist-info/entry_points.txt,sha256=F_DNUmGZU2Kme-8NsWM5LLE8piGMafYZygRYhOVtcjA,54
|
|
229
|
+
port_ocean-0.30.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|