fg-data-profiling 4.19.0__py2.py3-none-any.whl

data_profiling/config_minimal.yaml

@@ -0,0 +1,222 @@
+# Title of the document
+title: YData Profiling Report
+
+# Metadata
+dataset:
+  description: ""
+  creator: ""
+  author: ""
+  copyright_holder: ""
+  copyright_year: ""
+  url: ""
+
+variables:
+  descriptions: {}
+
+# infer dtypes
+infer_dtypes: false
+
+# Show the description at each variable (in addition to the overview tab)
+show_variable_description: true
+
+# Number of workers (0=multiprocessing.cpu_count())
+pool_size: 0
+
+# Show the progress bar
+progress_bar: true
+
+# Per variable type description settings
+vars:
+  num:
+    quantiles:
+      - 0.05
+      - 0.25
+      - 0.5
+      - 0.75
+      - 0.95
+    skewness_threshold: 20
+    low_categorical_threshold: 5
+    # Set to zero to disable
+    chi_squared_threshold: 0
+  cat:
+    length: false
+    characters: false
+    words: false
+    cardinality_threshold: 50
+    n_obs: 5
+    # Set to zero to disable
+    chi_squared_threshold: 0
+    coerce_str_to_date: false
+    redact: false
+    histogram_largest: 10
+    stop_words: []
+  bool:
+    n_obs: 3
+    # string to boolean mapping dict
+    mappings:
+      t: true
+      f: false
+      yes: true
+      no: false
+      y: true
+      n: false
+      "true": true
+      "false": false
+  path:
+    active: false
+  file:
+    active: false
+  image:
+    active: false
+    exif: false
+    hash: false
+  url:
+    active: false
+  timeseries:
+    active: false
+    autocorrelation: 0.7
+    lags:
+      - 1
+      - 7
+      - 12
+      - 24
+      - 30
+    significance: 0.05
+    pacf_acf_lag: 100
+
+# Sort the variables. Possible values: "ascending", "descending" or null (leaves original sorting)
+sort: null
+
+# which diagrams to show
+missing_diagrams:
+  bar: false
+  matrix: false
+  heatmap: false
+
+correlations:
+  pearson:
+    calculate: false
+    warn_high_correlations: true
+    threshold: 0.9
+  spearman:
+    calculate: false
+    warn_high_correlations: false
+    threshold: 0.9
+  kendall:
+    calculate: false
+    warn_high_correlations: false
+    threshold: 0.9
+  phi_k:
+    calculate: false
+    warn_high_correlations: false
+    threshold: 0.9
+  cramers:
+    calculate: false
+    warn_high_correlations: true
+    threshold: 0.9
+  auto:
+    calculate: false
+    warn_high_correlations: true
+    threshold: 0.9
+
+# Bivariate / Pairwise relations
+interactions:
+  targets: []
+  continuous: false
+
+# For categorical
+categorical_maximum_correlation_distinct: 100
+
+report:
+  precision: 10
+
+# Plot-specific settings
+plot:
+  # Image format (svg or png)
+  image_format: svg
+  dpi: 800
+
+  scatter_threshold: 1000
+
+  correlation:
+    cmap: RdBu
+    bad: "#000000"
+
+  missing:
+    cmap: RdBu
+    # Force labels when there are > 50 variables
+    force_labels: true
+
+  cat_frequency:
+    show: true  # if false, the category frequency plot is turned off
+    type: bar  # options: 'bar', 'pie'
+    max_unique: 0
+    colors: null  # use null for default or give a list of matplotlib recognized strings
+
+  histogram:
+    x_axis_labels: true
+
+    # Number of bins (set to 0 to automatically detect the bin size)
+    bins: 50
+
+    # Maximum number of bins (when bins=0)
+    max_bins: 250
+
+  font_path: null
+
+# The number of observations to show
+n_obs_unique: 5
+n_extreme_obs: 5
+n_freq_table_max: 10
+
+# Use `deep` flag for memory_usage
+memory_deep: false
+
+# Configuration related to the duplicates
+duplicates:
+  head: 0
+  key: "# duplicates"
+
+# Configuration related to the samples area
+samples:
+  head: 0
+  tail: 0
+  random: 0
+
+# Configuration related to the rejection of variables
+reject_variables: true
+
+# When in a Jupyter notebook
+notebook:
+  iframe:
+    height: 800px
+    width: 100%
+    # or 'src'
+    attribute: srcdoc
+
+html:
+  # Minify the html
+  minify_html: true
+
+  # Offline support
+  use_local_assets: true
+
+  # If true, single file, else directory with assets
+  inline: true
+
+  # Show navbar
+  navbar_show: true
+
+  # Assets prefix if inline = true
+  assets_prefix: null
+
+  # Styling options for the HTML report
+  style:
+    theme: null
+    logo: ""
+    primary_colors:
+      - "#0d6efd"
+      - "#dc3545"
+      - "#198754"
+
+  full_width: false

data_profiling/controller/__init__.py

@@ -0,0 +1 @@
+"""The controller module handles all user interaction with the package (console, jupyter, etc.)."""

data_profiling/controller/console.py

@@ -0,0 +1,125 @@
+"""This file add the console interface to the package."""
+import argparse
+from pathlib import Path
+from typing import Any, List, Optional
+
+from data_profiling.__init__ import ProfileReport, __version__
+from data_profiling.utils.dataframe import read_pandas
+
+
+def parse_args(args: Optional[List[Any]] = None) -> argparse.Namespace:
+    """Parse the command line arguments for the `data_profiling` binary.
+
+    Args:
+      args: List of input arguments. (Default value=None).
+
+    Returns:
+      Namespace with parsed arguments.
+
+    """
+    parser = argparse.ArgumentParser(
+        description="Profile the variables in a CSV file and generate a HTML report."
+    )
+
+    # Version
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+
+    # Console specific
+    parser.add_argument(
+        "-s",
+        "--silent",
+        help="Only generate but do not open report",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "-m",
+        "--minimal",
+        help="Minimal configuration for big data sets",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "-e",
+        "--explorative",
+        help="Explorative configuration featuring unicode, file and image analysis",
+        action="store_true",
+    )
+
+    # Config
+    parser.add_argument(
+        "--pool_size", type=int, default=0, help="Number of CPU cores to use"
+    )
+    parser.add_argument(
+        "--title",
+        type=str,
+        default="YData Profiling Report",
+        help="Title for the report",
+    )
+
+    parser.add_argument(
+        "--infer_dtypes",
+        default=False,
+        action="store_true",
+        help="To infer dtypes of the dataframe",
+    )
+
+    parser.add_argument(
+        "--no-infer_dtypes",
+        dest="infer_dtypes",
+        action="store_false",
+        help="To read dtypes as read by pandas",
+    )
+
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default=None,
+        help="Specify a yaml config file. Have a look at the 'config_default.yaml' as a starting point.",
+    )
+
+    parser.add_argument(
+        "input_file",
+        type=str,
+        help="CSV file (or other file type supported by pandas) to profile",
+    )
+    parser.add_argument(
+        "output_file",
+        type=str,
+        nargs="?",
+        help="Output report file. If empty, replaces the input_file's extension with .html and uses that.",
+        default=None,
+    )
+
+    return parser.parse_args(args)
+
+
+def main(args: Optional[List[Any]] = None) -> None:
+    """Run the `data_profiling` package.
+
+    Args:
+      args: Arguments for the programme (Default value=None).
+    """
+
+    # Parse the arguments
+    parsed_args = parse_args(args)
+    kwargs = vars(parsed_args)
+
+    input_file = Path(kwargs.pop("input_file"))
+    output_file = kwargs.pop("output_file")
+    if output_file is None:
+        output_file = str(input_file.with_suffix(".html"))
+
+    silent = kwargs.pop("silent")
+
+    # read the DataFrame
+    df = read_pandas(input_file)
+
+    # Generate the profiling report
+    p = ProfileReport(
+        df,
+        **kwargs,
+    )
+    p.to_file(Path(output_file), silent=silent)

data_profiling/controller/pandas_decorator.py

@@ -0,0 +1,21 @@
+"""This file add the decorator on the DataFrame object."""
+from pandas import DataFrame
+
+from data_profiling.profile_report import ProfileReport
+
+
+def profile_report(df: DataFrame, **kwargs) -> ProfileReport:
+    """Profile a DataFrame.
+
+    Args:
+        df: The DataFrame to profile.
+        **kwargs: Optional arguments for the ProfileReport object.
+
+    Returns:
+        A ProfileReport of the DataFrame.
+    """
+    p = ProfileReport(df, **kwargs)
+    return p
+
+
+DataFrame.profile_report = profile_report

data_profiling/expectations_report.py

@@ -0,0 +1,117 @@
+from typing import Any, Optional
+
+import pandas as pd
+from visions import VisionsTypeset
+
+from data_profiling.config import Settings
+from data_profiling.model import BaseDescription, expectation_algorithms
+from data_profiling.model.handler import Handler
+from data_profiling.utils.dataframe import slugify
+
+
+class ExpectationHandler(Handler):
+    """Default handler"""
+
+    def __init__(self, typeset: VisionsTypeset, *args, **kwargs):
+        mapping = {
+            "Unsupported": [expectation_algorithms.generic_expectations],
+            "Text": [expectation_algorithms.categorical_expectations],
+            "Categorical": [expectation_algorithms.categorical_expectations],
+            "Boolean": [expectation_algorithms.categorical_expectations],
+            "Numeric": [expectation_algorithms.numeric_expectations],
+            "URL": [expectation_algorithms.url_expectations],
+            "File": [expectation_algorithms.file_expectations],
+            "Path": [expectation_algorithms.path_expectations],
+            "DateTime": [expectation_algorithms.datetime_expectations],
+            "Image": [expectation_algorithms.image_expectations],
+        }
+        super().__init__(mapping, typeset, *args, **kwargs)
+
+
+class ExpectationsReport:
+    config: Settings
+    df: Optional[pd.DataFrame] = None
+
+    @property
+    def typeset(self) -> Optional[VisionsTypeset]:
+        return None
+
+    def to_expectation_suite(
+        self,
+        suite_name: Optional[str] = None,
+        data_context: Optional[Any] = None,
+        save_suite: bool = True,
+        run_validation: bool = True,
+        build_data_docs: bool = True,
+        handler: Optional[Handler] = None,
+    ) -> Any:
+        """
+        All parameters default to True to make it easier to access the full functionality of Great Expectations out of
+        the box.
+        Args:
+            suite_name: The name of your expectation suite
+            data_context: A user-specified data context
+            save_suite: Boolean to determine whether to save the suite to .json as part of the method
+            run_validation: Boolean to determine whether to run validation as part of the method
+            build_data_docs: Boolean to determine whether to build data docs, save the .html file, and open data docs in
+                your browser
+            handler: The handler to use for building expectation
+
+        Returns:
+            An ExpectationSuite
+        """
+        try:
+            import great_expectations as ge
+        except ImportError as ex:
+            raise ImportError(
+                "Please install great expectations before using the expectation functionality"
+            ) from ex
+
+        # Use report title if suite is empty
+        if suite_name is None:
+            suite_name = slugify(self.config.title)
+
+        # Use the default handler if none
+        if handler is None:
+            handler = ExpectationHandler(self.typeset)
+
+        # Obtain the ge context and create the expectation suite
+        if not data_context:
+            data_context = ge.data_context.DataContext()
+
+        suite = data_context.add_expectation_suite(suite_name)
+
+        # Instantiate an in-memory pandas dataset
+        batch = ge.dataset.PandasDataset(self.df, expectation_suite=suite)
+
+        # Obtain the profiling summary
+        summary: BaseDescription = self.get_description()  # type: ignore
+
+        # Dispatch to expectations per semantic variable type
+        for name, variable_summary in summary.variables.items():
+            handler.handle(variable_summary["type"], name, variable_summary, batch)
+
+        # We don't actually update the suite object on the batch in place, so need
+        # to get the populated suite from the batch
+        suite = batch.get_expectation_suite()
+
+        validation_result_identifier = None
+        if run_validation:
+            batch = ge.dataset.PandasDataset(self.df, expectation_suite=suite)
+
+            results = data_context.run_validation_operator(
+                "action_list_operator", assets_to_validate=[batch]
+            )
+            validation_result_identifier = results.list_validation_result_identifiers()[
+                0
+            ]
+
+        # Write expectations and open data docs
+        if save_suite or build_data_docs:
+            data_context.save_expectation_suite(suite)
+
+        if build_data_docs:
+            data_context.build_data_docs()
+            data_context.open_data_docs(validation_result_identifier)
+
+        return batch.get_expectation_suite()

data_profiling/model/__init__.py

@@ -0,0 +1,4 @@
+"""The model module handles all logic/calculations, e.g. calculate statistics, testing for special conditions."""
+from data_profiling.model.description import BaseAnalysis, BaseDescription
+
+__all__ = ["BaseAnalysis", "BaseDescription"]