acryl-datahub 1.3.0.1rc3__py3-none-any.whl → 1.3.0.1rc4__py3-none-any.whl

{acryl_datahub-1.3.0.1rc3.dist-info → acryl_datahub-1.3.0.1rc4.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.3.0.1rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.3.0.1rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=em6A_AK1PEbN80kEFbwFQP_4ZhxiWZZekBVZKx4EpV4,323
+datahub/_version.py,sha256=wuuyfFhYvCgNlVjy7dlOQ6sSJuFvaUAnTlhSVJzQ-fM,323
 datahub/entrypoints.py,sha256=VcbU6Z47b_JKW1zI-WJMYIngm05FSogKLiuvFNtyNcI,9088
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -476,7 +476,7 @@ datahub/ingestion/source/schema_inference/avro.py,sha256=aaqCMhLU2nxMJYPSNZv0o0A
 datahub/ingestion/source/schema_inference/base.py,sha256=dI98TOieCqqA1SdB6729EAReanGX2AC7UgSDkPls8Sg,379
 datahub/ingestion/source/schema_inference/csv_tsv.py,sha256=ypuBZEAf8Hx2Efrvu1nMWDdqVH_lg4i7N68YCwi8NiU,2259
 datahub/ingestion/source/schema_inference/json.py,sha256=p5S-3idn65V2uad5T8txs1UakA4cfllcrxfN-6qltss,2577
-datahub/ingestion/source/schema_inference/object.py,sha256=dhSOtxVJHbTDY0hWeHwdLYHnOsW07Omk7Y4DPeztie0,5847
+datahub/ingestion/source/schema_inference/object.py,sha256=ERR0XdiGE_qBWbNvt1oEWPYeB7ZNAsCnTZTF3ngn4F8,6582
 datahub/ingestion/source/schema_inference/parquet.py,sha256=CdqsNuiabLLCulWbuPMssijeFmKLv3M5MKFIhlatpWA,3456
 datahub/ingestion/source/sigma/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sigma/config.py,sha256=ztZf0YisGSXKgKeqP9ipDlRKLXU-Y-XABqm7HCJ8pvA,6265
@@ -646,12 +646,12 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_internal_schema_classes.py,sha256=1UZsNj9XmThYFXbG39BVKlaTTFywzayhVVon6svD3kM,1076970
-datahub/metadata/schema.avsc,sha256=P6j7fiukfv03ZW8gis3m3mVKGlSV2JhgMcmrtf5sU7Q,775491
+datahub/metadata/_internal_schema_classes.py,sha256=iJEKSTxDX1uH8YgUwxt0_HSd2AryK0-vUTBeN-7wG0Q,1076970
+datahub/metadata/schema.avsc,sha256=sn-1LEnMogcM9WhP7f8PWa_I7mgLALVeN75XtuCd4hY,812831
 datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
-datahub/metadata/_urns/urn_defs.py,sha256=_LgqKLHrmHHxpvrP-93NMJSLEnoFI8q72lkX17mK1XA,143257
+datahub/metadata/_urns/urn_defs.py,sha256=q1vWaLDYps2cfLh5IYsWUrbFYx_N2txLAFIl9vKdq-M,143257
 datahub/metadata/com/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/events/__init__.py,sha256=s_dR0plZF-rOxxIbE8ojekJqwiHzl2WYR-Z3kW6kKS0,298
@@ -1128,8 +1128,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.3.0.1rc3.dist-info/METADATA,sha256=Q32VN8kEGo-T0nge3wOkp_EmXJQQZKtZYl9SnsCu3PY,184504
-acryl_datahub-1.3.0.1rc3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.3.0.1rc3.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.3.0.1rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.3.0.1rc3.dist-info/RECORD,,
+acryl_datahub-1.3.0.1rc4.dist-info/METADATA,sha256=cJb6Hz3UpjAA15FINI4riQLYt7C5Q-aUsoWz2pelBp0,184504
+acryl_datahub-1.3.0.1rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.3.0.1rc4.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.3.0.1rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.3.0.1rc4.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.3.0.1rc3"
+__version__ = "1.3.0.1rc4"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/source/schema_inference/object.py

@@ -1,4 +1,4 @@
-from collections import Counter
+from collections import Counter, defaultdict
 from typing import Any, Counter as CounterType, Dict, Sequence, Tuple, Union
 
 from typing_extensions import TypedDict
@@ -84,7 +84,7 @@ def is_nullable_collection(
 
 
 def construct_schema(
-    collection: Sequence[Dict[str, Any]], delimiter: str
+    collection: Sequence[Dict[str, Any]], delimiter: str = "."
 ) -> Dict[Tuple[str, ...], SchemaDescription]:
     """
     Construct (infer) a schema from a collection of documents.
@@ -104,9 +104,11 @@ def construct_schema(
             string to concatenate field names by
     """
 
-    schema: Dict[Tuple[str, ...], BasicSchemaDescription] = {}
+    schema: Dict[Tuple[str, ...], BasicSchemaDescription] = defaultdict(
+        lambda: {"types": Counter(), "count": 0}
+    )
 
-    def append_to_schema(doc: Dict[str, Any], parent_prefix: Tuple[str, ...]) -> None:
+    def append_to_schema(doc: Dict[str, Any], parent_prefix: Tuple[str, ...]) -> int:
         """
         Recursively update the schema with a document, which may/may not contain nested fields.
 
@@ -118,18 +120,24 @@ def construct_schema(
                 prefix of fields that the document is under, pass an empty tuple when initializing
         """
 
+        # we want to make sure that parents of nested structures are included first, before their children, so that
+        # they are displayed properly in the UI, also in the event of trimming the list (which happens, for example,
+        # in mongodb ingestor)
+        max_count = 0
         for key, value in doc.items():
             new_parent_prefix = parent_prefix + (key,)
 
             # if nested value, look at the types within
             if isinstance(value, dict):
-                append_to_schema(value, new_parent_prefix)
+                max_count = max(append_to_schema(value, new_parent_prefix), max_count)
             # if array of values, check what types are within
             if isinstance(value, list):
                 for item in value:
                     # if dictionary, add it as a nested object
                     if isinstance(item, dict):
-                        append_to_schema(item, new_parent_prefix)
+                        max_count = max(
+                            append_to_schema(item, new_parent_prefix), max_count
+                        )
 
             # don't record None values (counted towards nullable)
             if value is not None:
@@ -143,6 +151,14 @@ def construct_schema(
                     # update the type count
                     schema[new_parent_prefix]["types"].update({type(value): 1})
                     schema[new_parent_prefix]["count"] += 1
+            max_count = max(schema[new_parent_prefix]["count"], max_count)
+
+        if parent_prefix != ():
+            schema[parent_prefix]["count"] = max(
+                schema[parent_prefix]["count"], max_count
+            )
+
+        return max_count
 
     for document in collection:
         append_to_schema(document, ())