arthur-common 1.0.1__py3-none-any.whl

arthur_common/tools/schema_inferer.py

@@ -0,0 +1,104 @@
+from typing import Any
+from uuid import uuid4
+
+import pandas as pd
+from arthur_common.models.schema_definitions import (
+    DatasetColumn,
+    DatasetListType,
+    DatasetObjectType,
+    DatasetScalarType,
+    DatasetSchema,
+    DType,
+    ScopeSchemaTag,
+)
+from arthur_common.tools.duckdb_data_loader import DuckDBOperator, escape_identifier
+
+
+class SchemaInferer:
+    def __init__(self, data: list[dict[str, Any]] | pd.DataFrame):
+        self.conn = DuckDBOperator.load_data_to_duckdb(
+            data,
+            preprocess_schema=True,
+            table_name="root",
+        )
+
+    def infer_schema(self) -> DatasetSchema:
+        columns = self._infer_schema()
+        schema = DatasetSchema(columns=[], alias_mask={})
+        for key, definition in columns.object.items():
+            schema.columns.append(
+                DatasetColumn(id=uuid4(), source_name=key, definition=definition),
+            )
+        # Close connection to destroy all temp tables and free up memory
+        self.conn.close()
+        return schema
+
+    def _infer_nested_schema(self, col_name: str, table: str) -> DatasetObjectType:
+        escaped_col = escape_identifier(col_name)
+        self.conn.sql(
+            f"CREATE OR REPLACE TEMP TABLE {escaped_col} AS SELECT UNNEST({escaped_col}) as {escaped_col} FROM {table}",
+        )
+        return self._infer_schema(escaped_col)
+
+    def _infer_schema(self, table: str = "root") -> DatasetObjectType:
+        ddb_schema: list[tuple[Any, Any, Any]] = self.conn.sql(
+            f"DESCRIBE {table}",
+        ).fetchall()
+
+        obj = DatasetObjectType(id=uuid4(), object={}, nullable=False)
+        timestamp_cols = []
+
+        for column in ddb_schema:
+            col_type, col_name, col_nullable = (
+                str(column[1]),
+                str(column[0]),
+                str(column[2]) == "YES",
+            )
+            col_is_list = col_type[-2:] == "[]"
+            col_type = col_type.replace("[]", "")
+
+            # Handle structs / lists recursively
+            if col_is_list:
+                schema = self._infer_nested_schema(col_name, table)
+                obj.object[col_name] = DatasetListType(
+                    id=uuid4(),
+                    items=schema[col_name],
+                    nullable=col_nullable,
+                )
+            elif "STRUCT" in col_type:
+                schema = self._infer_nested_schema(col_name, table)
+                schema.nullable = col_nullable
+                obj.object[col_name] = schema
+            else:
+                scalar_schema = DatasetScalarType(id=uuid4(), dtype=DType.UNDEFINED)
+                match col_type:
+                    case "UUID":
+                        scalar_schema.dtype = DType.UUID
+                    case "VARCHAR":
+                        scalar_schema.dtype = DType.STRING
+                    case "BIGINT" | "INTEGER":
+                        scalar_schema.dtype = DType.INT
+                    case "DOUBLE" | "FLOAT":
+                        scalar_schema.dtype = DType.FLOAT
+                    case "BOOLEAN":
+                        scalar_schema.dtype = DType.BOOL
+                    case "JSON":
+                        # keep duckDB's json type in case the customer's data doesn't fit in well-structured types
+                        # an example is a JSON list like ["str", 0.234], because arrays can only have a single type
+                        # in duckDB
+                        scalar_schema.dtype = DType.JSON
+                    case "DATE":
+                        scalar_schema.dtype = DType.DATE
+                    case "TIMESTAMP_NS" | "TIMESTAMP WITH TIME ZONE" | "TIMESTAMP":
+                        scalar_schema.dtype = DType.TIMESTAMP
+                        timestamp_cols.append(scalar_schema)
+                    case _:
+                        raise NotImplementedError(f"Type {col_type} not mappable.")
+                obj.object[col_name] = scalar_schema
+
+        # auto assign primary timestamp tag if there's only one timestamp column
+        if len(timestamp_cols) == 1:
+            timestamp_col = timestamp_cols[0]
+            timestamp_col.tag_hints.append(ScopeSchemaTag.PRIMARY_TIMESTAMP)
+
+        return obj

arthur_common/tools/time_utils.py

@@ -0,0 +1,33 @@
+from datetime import datetime, timedelta
+
+STRFTIME_CODES_TO_TIMEDELTA = {
+    "%H": timedelta(hours=1),  # Hour (24-hour clock) as zero-padded decimal number
+    "%-H": timedelta(hours=1),  # Hour (24-hour clock) as decimal number
+    "%I": timedelta(hours=1),  # Hour (12-hour clock)
+    "%d": timedelta(days=1),  # Day of month as zero-padded decimal
+    "%-d": timedelta(days=1),  # Day of month as decimal number
+    "%j": timedelta(
+        days=1,
+    ),  # Day of year as zero-padded decimal number (001, ..., 366)
+    "%-j": timedelta(days=1),  # Day of year as decimal number (1, ..., 366)
+}
+
+
+def find_smallest_timedelta(format_string: str) -> timedelta | None:
+    """Scans a linux strftime format string to find the smallest time unit present.
+    Only considers units larger than or equal to an hour or less than or equal to a day.
+    Example: '%Y-%m-%d_%H' -> timedelta(hours=1), '%Y-%m-%d' -> timedelta(days=1)
+    """
+    smallest_delta = None
+    for code, delta in STRFTIME_CODES_TO_TIMEDELTA.items():
+        if code in format_string:
+            if smallest_delta is None or delta < smallest_delta:
+                smallest_delta = delta
+    return smallest_delta
+
+
+def check_datetime_tz_aware(dt: datetime) -> bool:
+    """Returns true if dt is timezone-aware, false if naive."""
+    return (
+        True if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None else False
+    )

arthur_common-1.0.1.dist-info/METADATA

@@ -0,0 +1,74 @@
+Metadata-Version: 2.3
+Name: arthur-common
+Version: 1.0.1
+Summary: Utility code common to Arthur platform components.
+License: MIT
+Author: Arthur
+Author-email: engineering@arthur.ai
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: datasketches (>=5.1.0)
+Requires-Dist: duckdb (>=1.1.3)
+Requires-Dist: fastapi (>=0.115.8)
+Requires-Dist: fsspec (>=2024.10.0)
+Requires-Dist: pandas (>=2.2.2)
+Requires-Dist: pydantic (>=2)
+Requires-Dist: tokencost (==0.1.21)
+Requires-Dist: types-python-dateutil (>=2.9.0)
+Requires-Dist: types-requests (>=2.32.0.20241016)
+Requires-Dist: typing-extensions (>=4.7.1)
+Description-Content-Type: text/markdown
+
+# Arthur Common
+
+Arthur Common is a library that contains common operations between Arthur platform services.
+
+## Installation
+
+To install the package, use [Poetry](https://python-poetry.org/):
+
+```bash
+poetry add arthur-common
+```
+
+or pip
+
+```bash
+pip install arthur-common
+```
+
+## Requirements
+
+- Python 3.12
+
+## Development
+
+To set up the development environment, ensure you have [Poetry](https://python-poetry.org/) installed, then run:
+
+```bash
+poetry env use 3.12
+poetry install
+```
+
+### Running Tests
+
+This project uses [pytest](https://pytest.org/) for testing. To run the tests, execute:
+
+```bash
+poetry run pytest
+```
+
+## License
+
+This project is licensed under the MIT License.
+
+## Authors
+
+- Arthur <engineering@arthur.ai>
+
+# ALEX
+- Change for testing
+

arthur_common-1.0.1.dist-info/RECORD

@@ -0,0 +1,40 @@
+arthur_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/__version__.py,sha256=d4QHYmS_30j0hPN8NmNPnQ_Z0TphDRbu4MtQj9cT9e8,22
+arthur_common/aggregations/__init__.py,sha256=vISWyciQAtksa71OKeHNP-QyFGd1NzBKq_LBsG0QSG8,67
+arthur_common/aggregations/aggregator.py,sha256=cw6mr1Dl0sH7Rn_EggCRvduC7GvC8XTDP2L84FODKY0,7445
+arthur_common/aggregations/functions/README.md,sha256=MkZoTAJ94My96R5Z8GAxud7S6vyR0vgVi9gqdt9a4XY,5460
+arthur_common/aggregations/functions/__init__.py,sha256=HqC3UNRURX7ZQHgamTrQvfA8u_FiZGZ4I4eQW7Ooe5o,1299
+arthur_common/aggregations/functions/categorical_count.py,sha256=_hk1TYkrnCAe-0Hflt4W3Nvrp-c-vWHSgVxaqcFNWAI,3368
+arthur_common/aggregations/functions/confusion_matrix.py,sha256=xqJkwcFZck_F1tsvMeJdmU-EOSOfWJS_N7-PXYJiSmo,16351
+arthur_common/aggregations/functions/inference_count.py,sha256=BGWa262UxpkqY5Y_Pm22URSRvwXzdRbGUcd4DSVJbN0,2312
+arthur_common/aggregations/functions/inference_count_by_class.py,sha256=BwZw8wCmFHRoau5oDgcDAkcwNhXjktFBCG1Sfsj6iGY,7830
+arthur_common/aggregations/functions/inference_null_count.py,sha256=wZDz89_23bGjB_Tb3ob_69_VhugFxzbSRkuENfyJ-ic,2867
+arthur_common/aggregations/functions/mean_absolute_error.py,sha256=rDEQlKEDyy_zRewtgthB_BK2oKrW6ymWmdtkxUFzfW8,4233
+arthur_common/aggregations/functions/mean_squared_error.py,sha256=TMUyPPPEHuG9QFmD2gZxmxy-f_CDU4ds_2R5-DFr42c,4251
+arthur_common/aggregations/functions/multiclass_confusion_matrix.py,sha256=QM27MhZMvF5Q5yxENSsHx_MMV_5h6eiRPP554jsdBqY,8204
+arthur_common/aggregations/functions/multiclass_inference_count_by_class.py,sha256=dmf4OI4KcFVvE61S2TnPZvBlNOhlFNFKrV7x2_ys5ZU,3201
+arthur_common/aggregations/functions/numeric_stats.py,sha256=eN1q10KquF8GeBOOktBuatV9Zxvd_--ZzklqzDpT9qw,3124
+arthur_common/aggregations/functions/numeric_sum.py,sha256=DO7jbmUEnyyJliEtYLHb58SDCVBTsFKvywFnADXFpw8,3036
+arthur_common/aggregations/functions/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/aggregations/functions/shield_aggregations.py,sha256=nkMlj9V7NIKeRP46jpsrlfB741RHk2CgwimD7BYp9To,31540
+arthur_common/aggregations/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/models/connectors.py,sha256=Ng2hwXQumfLE9SYepYZs2L5Y6aAIXAQfkJIa0rKCvWQ,1565
+arthur_common/models/datasets.py,sha256=giG_8mv_3ilBf7cIvRV0_TDCDdb4qxRbYZvl7hRb6l8,491
+arthur_common/models/metrics.py,sha256=gFEGuM4kuac2CqpPN69BM2cWG-SUPE4-1jZVHv_M3M0,8380
+arthur_common/models/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/models/schema_definitions.py,sha256=MiM-oynqh71zZvVyM-XF5nN55dgOeqEajVyL8ZE3Wuo,14571
+arthur_common/models/shield.py,sha256=1ZblfULKCf5BEvYURO5WScyfmijGwjAmcj0XADlF-XY,19110
+arthur_common/models/task_job_specs.py,sha256=GLJ7qmrb5eXnl5PiV27nnx_yG4S4sc4NDJ8-6xmNDLM,2796
+arthur_common/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/tools/aggregation_analyzer.py,sha256=e4F8vsYDRRTzUmNVIl1vFrn9_nEeYDYcP3ygk7i1964,9534
+arthur_common/tools/aggregation_loader.py,sha256=3CF46bNi-GdJBNOXkjYfCQ1Aung8lf65L532sdWmR_s,2351
+arthur_common/tools/duckdb_data_loader.py,sha256=XrdXRFkgiGtYulOGsC4khVf12sNiSFx5hB5vD7vQzFE,11066
+arthur_common/tools/functions.py,sha256=FWL4eWO5-vLp86WudT-MGUKvf2B8f02IdoXQFKd6d8k,1093
+arthur_common/tools/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arthur_common/tools/schema_inferer.py,sha256=PkAOHZRk_rZ1OZSigYrfzH-jERb9B_Gu7pOMl9WJQA8,4202
+arthur_common/tools/time_utils.py,sha256=4gfiu9NXfvPZltiVNLSIQGylX6h2W0viNi9Kv4bKyfw,1410
+arthur_common-1.0.1.dist-info/METADATA,sha256=WrMAk42ZfTrZhAz12r127fJhuglMx-zHnDXPMp7tzsk,1596
+arthur_common-1.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+arthur_common-1.0.1.dist-info/RECORD,,

arthur_common-1.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any