wedata-feature-engineering 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

feature_store/training_set_client/training_set_client.py

@@ -1,196 +0,0 @@
-import json
-import logging
-import os
-from collections import defaultdict
-from types import ModuleType
-from typing import Any, Dict, List, Optional, Set, Union
-
-import mlflow
-import yaml
-from mlflow.models import Model, ModelSignature
-from mlflow.utils.file_utils import TempDir, YamlSafeDumper, read_yaml
-from pyspark.sql import DataFrame
-from pyspark.sql.functions import struct
-
-from feature_store.entities.feature_column_info import FeatureColumnInfo
-from feature_store.entities.feature_function import FeatureFunction
-from feature_store.entities.feature_lookup import FeatureLookup
-from feature_store.entities.feature_spec import FeatureSpec
-from feature_store.entities.training_set import TrainingSet
-from feature_store.spark_client.spark_client import SparkClient
-
-from feature_store.constants.constants import (
-    _NO_RESULT_TYPE_PASSED,
-    _PREBUILT_ENV_URI,
-    _USE_SPARK_NATIVE_JOIN,
-    _WARN,
-    MODEL_DATA_PATH_ROOT,
-    PREDICTION_COLUMN_NAME,
-)
-
-from feature_store.utils import common_utils, training_set_utils
-from feature_store.utils.feature_spec_utils import convert_to_yaml_string
-
-_logger = logging.getLogger(__name__)
-
-FEATURE_SPEC_GRAPH_MAX_COLUMN_INFO = 1000
-
-
-class TrainingSetClient:
-    def __init__(
-        self,
-        spark_client: SparkClient
-    ):
-        self._spark_client = spark_client
-
-    def create_training_set(
-        self,
-        feature_spec: FeatureSpec,
-        feature_column_infos: List[FeatureColumnInfo],
-        label_names: List[str],
-        df: DataFrame,
-        ft_metadata: training_set_utils._FeatureTableMetadata,
-        kwargs,
-    ):
-        uc_function_infos = training_set_utils.get_uc_function_infos(
-            self._spark_client,
-            {odci.udf_name for odci in feature_spec.on_demand_column_infos},
-        )
-
-        # TODO(divyagupta-db): Move validation from _validate_join_feature_data in feature_lookup_utils.py
-        #  to a helper function called here and in score_batch.
-
-        # Add consumer of each feature and instrument as final step
-        consumer_feature_table_map = defaultdict(list)
-        for feature in feature_column_infos:
-            consumer_feature_table_map[feature.table_name].append(feature.feature_name)
-        consumed_udf_names = [f.udf_name for f in feature_spec.function_infos]
-
-        # Spark query planning is known to cause spark driver to crash if there are many feature tables to PiT join.
-        # See https://docs.google.com/document/d/1EyA4vvlWikTJMeinsLkxmRAVNlXoF1eqoZElOdqlWyY/edit
-        # So we disable native join by default.
-        training_set_utils.warn_if_non_photon_for_native_spark(
-            kwargs.get(_USE_SPARK_NATIVE_JOIN, False), self._spark_client
-        )
-        return TrainingSet(
-            feature_spec,
-            df,
-            label_names,
-            ft_metadata.feature_table_metadata_map,
-            ft_metadata.feature_table_data_map,
-            uc_function_infos,
-            kwargs.get(_USE_SPARK_NATIVE_JOIN, False),
-        )
-
-    def create_training_set_from_feature_lookups(
-        self,
-        df: DataFrame,
-        feature_lookups: List[Union[FeatureLookup, FeatureFunction]],
-        label: Union[str, List[str], None],
-        exclude_columns: List[str],
-        **kwargs,
-    ) -> TrainingSet:
-
-        # 获取特征查找列表和特征函数列表
-        features = feature_lookups
-        feature_lookups = [f for f in features if isinstance(f, FeatureLookup)]
-        feature_functions = [f for f in features if isinstance(f, FeatureFunction)]
-
-        # 如果未提供标签，则用空列表初始化label_names
-        label_names = common_utils.as_list(label, [])
-        del label
-
-        # 校验数据集和标签
-        training_set_utils.verify_df_and_labels(df, label_names, exclude_columns)
-
-        # 获取特征表元数据
-        ft_metadata = training_set_utils.get_table_metadata(
-            self._spark_client,
-            {fl.table_name for fl in feature_lookups}
-        )
-
-        column_infos = training_set_utils.get_column_infos(
-            feature_lookups,
-            feature_functions,
-            ft_metadata,
-            df_columns=df.columns,
-            label_names=label_names,
-        )
-
-        training_set_utils.validate_column_infos(
-            self._spark_client,
-            ft_metadata,
-            column_infos.source_data_column_infos,
-            column_infos.feature_column_infos,
-            column_infos.on_demand_column_infos,
-            label_names,
-        )
-
-        # Build feature_spec locally for comparison with the feature spec yaml generated by the
-        # FeatureStore backend. This will be removed once the migration is validated.
-        feature_spec = training_set_utils.build_feature_spec(
-            feature_lookups,
-            ft_metadata,
-            column_infos,
-            exclude_columns
-        )
-
-        return self.create_training_set(
-            feature_spec,
-            column_infos.feature_column_infos,
-            label_names,
-            df,
-            ft_metadata,
-            kwargs=kwargs,
-        )
-
-
-
-
-
-    def create_feature_spec(
-        self,
-        name: str,
-        features: List[Union[FeatureLookup, FeatureFunction]],
-        sparkClient: SparkClient,
-        exclude_columns: List[str] = [],
-    ) -> FeatureSpec:
-
-        feature_lookups = [f for f in features if isinstance(f, FeatureLookup)]
-        feature_functions = [f for f in features if isinstance(f, FeatureFunction)]
-
-        # Maximum of 100 FeatureFunctions is supported
-        if len(feature_functions) > training_set_utils.MAX_FEATURE_FUNCTIONS:
-            raise ValueError(
-                f"A maximum of {training_set_utils.MAX_FEATURE_FUNCTIONS} FeatureFunctions are supported."
-            )
-
-        # Get feature table metadata and column infos
-        ft_metadata = training_set_utils.get_table_metadata(
-            self._spark_client,
-            {fl.table_name for fl in feature_lookups}
-        )
-        column_infos = training_set_utils.get_column_infos(
-            feature_lookups,
-            feature_functions,
-            ft_metadata,
-        )
-
-        column_infos = training_set_utils.add_inferred_source_columns(column_infos)
-
-        training_set_utils.validate_column_infos(
-            self._spark_client,
-            ft_metadata,
-            column_infos.source_data_column_infos,
-            column_infos.feature_column_infos,
-            column_infos.on_demand_column_infos,
-        )
-
-        feature_spec = training_set_utils.build_feature_spec(
-            feature_lookups,
-            ft_metadata,
-            column_infos,
-            exclude_columns
-        )
-
-        return feature_spec

feature_store/utils/common_utils.py

@@ -1,96 +0,0 @@
-"""
-通用工具函数
-"""
-
-from collections import Counter
-from typing import Any, List
-
-from mlflow.store.artifact.models_artifact_repo import ModelsArtifactRepository
-from mlflow.store.artifact.runs_artifact_repo import RunsArtifactRepository
-
-
-def is_artifact_uri(uri):
-    """
-    Checks the artifact URI is associated with a MLflow model or run.
-    The actual URI can be a model URI, model URI + subdirectory, or model URI + path to artifact file.
-    """
-    return ModelsArtifactRepository.is_models_uri(
-        uri
-    ) or RunsArtifactRepository.is_runs_uri(uri)
-
-def as_list(obj, default=None):
-    if not obj:
-        return default
-    elif isinstance(obj, list):
-        return obj
-    else:
-        return [obj]
-
-def get_duplicates(elements: List[Any]) -> List[Any]:
-    """
-    Returns duplicate elements in the order they first appear.
-    """
-    element_counts = Counter(elements)
-    duplicates = []
-    for e in element_counts.keys():
-        if element_counts[e] > 1:
-            duplicates.append(e)
-    return duplicates
-
-def validate_strings_unique(strings: List[str], error_template: str):
-    """
-    Validates all strings are unique, otherwise raise ValueError with the error template and duplicates.
-    Passes single-quoted, comma delimited duplicates to the error template.
-    """
-    duplicate_strings = get_duplicates(strings)
-    if duplicate_strings:
-        duplicates_formatted = ", ".join([f"'{s}'" for s in duplicate_strings])
-        raise ValueError(error_template.format(duplicates_formatted))
-
-def sanitize_identifier(identifier: str):
-    """
-    Sanitize and wrap an identifier with backquotes. For example, "a`b" becomes "`a``b`".
-    Use this function to sanitize identifiers such as column names in SQL and PySpark.
-    """
-    return f"`{identifier.replace('`', '``')}`"
-
-
-def sanitize_identifiers(identifiers: List[str]):
-    """
-    Sanitize and wrap the identifiers in a list with backquotes.
-    """
-    return [sanitize_identifier(i) for i in identifiers]
-
-
-def sanitize_multi_level_name(multi_level_name: str):
-    """
-    Sanitize a multi-level name (such as an Unity Catalog table name) by sanitizing each segment
-    and joining the results. For example, "ca+t.fo`o.ba$r" becomes "`ca+t`.`fo``o`.`ba$r`".
-    """
-    segments = multi_level_name.split(".")
-    return ".".join(sanitize_identifiers(segments))
-
-
-def unsanitize_identifier(identifier: str):
-    """
-    Unsanitize an identifier. Useful when we get a possibly sanitized identifier from Spark or
-    somewhere else, but we need an unsanitized one.
-    Note: This function does not check the correctness of the identifier passed in. e.g. `foo``
-    is not a valid sanitized identifier. When given such invalid input, this function returns
-    invalid output.
-    """
-    if len(identifier) >= 2 and identifier[0] == "`" and identifier[-1] == "`":
-        return identifier[1:-1].replace("``", "`")
-    else:
-        return identifier
-
-
-# strings containing \ or ' can break sql statements, so escape them.
-def escape_sql_string(input_str: str) -> str:
-    return input_str.replace("\\", "\\\\").replace("'", "\\'")
-
-def get_unique_list_order(elements: List[Any]) -> List[Any]:
-    """
-    Returns unique elements in the order they first appear.
-    """
-    return list(dict.fromkeys(elements))

wedata_feature_engineering-0.1.4.dist-info/RECORD

@@ -1,41 +0,0 @@
-feature_store/__init__.py,sha256=CP3YAMoy3pSTWRYzTza_CYBnGbTv_KzycVEBMQCeiD8,101
-feature_store/client.py,sha256=FG1xK460rD859iSY4VA75XeYhqStJD8Wlr0sRxk25LI,5267
-feature_store/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/constants/constants.py,sha256=exW3kiFLDyCmU9cYHFjcvIQhPWEpFtkogLXeB9Arfd8,827
-feature_store/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/entities/column_info.py,sha256=WezowI46YHDym5ZlbhCJDqhKbVcjXjnjt7dQdy3XqYM,4164
-feature_store/entities/data_type.py,sha256=VpHS6Fr3TphQQ8NbAcEnDJ-8eOZV6ivYuWxv3pAM2RM,3394
-feature_store/entities/environment_variables.py,sha256=ZEFml5H9MQuzBKM074mUrFYu-Sga4Knmxqiwpke2WGc,1679
-feature_store/entities/feature.py,sha256=wX8fTBlJq3GYdj9rrBDCY3kFgcVBBAiOOZdxEhnQkNQ,1241
-feature_store/entities/feature_column_info.py,sha256=-TGxRafYUaNKe0YzHus2XbfRaVrMv7pcffMdbtTT4nA,2031
-feature_store/entities/feature_function.py,sha256=R17INrCE-U_Uj9KLbFz69aYlOkTETTwQHMMo470F4lQ,1865
-feature_store/entities/feature_lookup.py,sha256=zUDMdDIboitOffYRZlurf_O_4UeBPmE5YS0PyCS2Fqg,7912
-feature_store/entities/feature_spec.py,sha256=F4MiKEyvKZSBh6Uv7V4vVLbamZ9fRClaC3HCrUeynDE,20079
-feature_store/entities/feature_spec_constants.py,sha256=YWDBfRiNDe6fUJFUBo3V4WYg2xsljoPAE-ZejfFZCgM,785
-feature_store/entities/feature_table.py,sha256=4ghopIvJcoIlyFiSEuTkOcDWn88c1Kt6q5LWM4BYEHI,6073
-feature_store/entities/feature_table_info.py,sha256=2vUaVdW_jw1dRAlmJWvBRueuMeuqWu_NYB9SlxLI7Uw,1126
-feature_store/entities/function_info.py,sha256=l0kmiq2R_QNfSMJ7y0xZohlMiemgYSr1dN5vzV8ijIs,7314
-feature_store/entities/on_demand_column_info.py,sha256=Eh5ieaj1TxC7DG6ipBZzH2ZyY0bwkLrDOkuZjgYr4gY,1297
-feature_store/entities/source_data_column_info.py,sha256=a9jQOJvehwDIrKPwsP6W9YRBSPNK2nZYypE6-p80CwA,542
-feature_store/entities/training_set.py,sha256=9H2uGnUxTAsk93Om50QxRELbeFCocwGMze2VexPVJWI,5569
-feature_store/feature_table_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/feature_table_client/feature_table_client.py,sha256=uir33K7oigrSnjTT6VbNOp0Nb22-X3JHd1_92kWjrow,10754
-feature_store/spark_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/spark_client/spark_client.py,sha256=vd-NCE9IGC0Ygqr-QSVY0teuWsQSkq_BFV4Mn6xMMNU,11578
-feature_store/training_set_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/training_set_client/training_set_client.py,sha256=Aa80xVXVE1KBdgplL9qqR8ftD5A5r2pfBttAhmySrB0,6696
-feature_store/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-feature_store/utils/common_utils.py,sha256=ck8pJYeN6vrcZmTrcnmIOOJWzYZaY3ZjvRSVme4tplo,3314
-feature_store/utils/feature_lookup_utils.py,sha256=iILSP4AFHXrjNTuId6mT7wtMFAsZejyxThr_mZHPRF4,22330
-feature_store/utils/feature_spec_utils.py,sha256=jeWzEhmkVW-bMRySMx_5grepHAlLquMhYxpbbiaJR-g,11582
-feature_store/utils/feature_utils.py,sha256=8KhlkWax3KAi_xRnStVPlhCxeUHO08VW2fmT9jN8QUs,2761
-feature_store/utils/schema_utils.py,sha256=8NhNUsF4Z6UtmzFeaVBnmb7xut0LqZepK3M27PSEpfE,4484
-feature_store/utils/topological_sort.py,sha256=ebzKxmxeCLk9seB1zR0ASCGXsZsa-DjxJeTc4KUadtg,6475
-feature_store/utils/training_set_utils.py,sha256=V5yW-XQ9in7gNOo4xsWy7txnSw_Z9Zxm4mV7MQmrWnk,22466
-feature_store/utils/uc_utils.py,sha256=ets7YlrAtkhW9kKyYajDNo6iZasBIhFyxUT2MOyLuV8,10767
-feature_store/utils/utils.py,sha256=T6dOUX3oOYRsbvXyTIElFZ20kNO92KMYPUCrqY5eomE,8953
-feature_store/utils/validation_utils.py,sha256=FslvrNs3kstqvM6THScLOluEE6O9RWlDrD9xiihTzlw,1735
-wedata_feature_engineering-0.1.4.dist-info/METADATA,sha256=uwmHZ4fVVcncF5YH_p3kUG24D377eKLraAcOlx-KU5o,493
-wedata_feature_engineering-0.1.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-wedata_feature_engineering-0.1.4.dist-info/top_level.txt,sha256=15761LgVdJ7tJWbdlYk0EZ560G9k6C4TE42dfLx8d0I,14
-wedata_feature_engineering-0.1.4.dist-info/RECORD,,

wedata_feature_engineering-0.1.4.dist-info/top_level.txt

@@ -1 +0,0 @@
-feature_store