oracle-ads 2.11.6__py3-none-any.whl → 2.11.8__py3-none-any.whl

ads/opctl/operator/lowcode/forecast/model/prophet.py

@@ -82,7 +82,7 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
 
             data = self.preprocess(df, series_id)
             data_i = self.drop_horizon(data)
-            if self.loaded_models is not None:
+            if self.loaded_models is not None and series_id in self.loaded_models:
                 model = self.loaded_models[series_id]
             else:
                 if self.perform_tuning:
@@ -133,9 +133,6 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
             }
 
     def _build_model(self) -> pd.DataFrame:
-        from prophet import Prophet
-        from prophet.diagnostics import cross_validation, performance_metrics
-
         full_data_dict = self.datasets.get_data_by_series()
         self.models = dict()
         self.outputs = dict()
@@ -160,6 +157,9 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
         return self.forecast_output.get_forecast_long()
 
     def run_tuning(self, data_i, model_kwargs_i):
+        from prophet import Prophet
+        from prophet.diagnostics import cross_validation, performance_metrics
+
         def objective(trial):
             params = {
                 "seasonality_mode": trial.suggest_categorical(
@@ -243,68 +243,70 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
         return model_kwargs_i
 
     def _generate_report(self):
-        import datapane as dp
+        import report_creator as rc
         from prophet.plot import add_changepoints_to_plot
 
-        series_ids = self.datasets.list_series_ids()
-
-        sec1_text = dp.Text(
-            "## Forecast Overview \n"
-            "These plots show your forecast in the context of historical data."
-        )
-        sec1 = _select_plot_list(
-            lambda s_id: self.models[s_id].plot(
-                self.outputs[s_id], include_legend=True
-            ),
-            series_ids=series_ids,
-        )
-
-        sec2_text = dp.Text(f"## Forecast Broken Down by Trend Component")
-        sec2 = _select_plot_list(
-            lambda s_id: self.models[s_id].plot_components(self.outputs[s_id]),
-            series_ids=series_ids,
-        )
+        series_ids = self.models.keys()
+        all_sections = []
+        if len(series_ids) > 0:
+            sec1 = _select_plot_list(
+                lambda s_id: self.models[s_id].plot(
+                    self.outputs[s_id], include_legend=True
+                ),
+                series_ids=series_ids,
+            )
+            section_1 = rc.Block(
+                rc.Heading("Forecast Overview", level=2),
+                rc.Text(
+                    "These plots show your forecast in the context of historical data."
+                ),
+                sec1,
+            )
 
-        sec3_text = dp.Text(f"## Forecast Changepoints")
-        sec3_figs = {
-            s_id: self.models[s_id].plot(self.outputs[s_id]) for s_id in series_ids
-        }
-        for s_id in series_ids:
-            add_changepoints_to_plot(
-                sec3_figs[s_id].gca(), self.models[s_id], self.outputs[s_id]
+            sec2 = _select_plot_list(
+                lambda s_id: self.models[s_id].plot_components(self.outputs[s_id]),
+                series_ids=series_ids,
+            )
+            section_2 = rc.Block(
+                rc.Heading("Forecast Broken Down by Trend Component", level=2), sec2
             )
-        sec3 = _select_plot_list(lambda s_id: sec3_figs[s_id], series_ids=series_ids)
-
-        all_sections = [sec1_text, sec1, sec2_text, sec2, sec3_text, sec3]
-
-        sec5_text = dp.Text(f"## Prophet Model Seasonality Components")
-        model_states = []
-        for s_id in series_ids:
-            m = self.models[s_id]
-            model_states.append(
-                pd.Series(
-                    m.seasonalities,
-                    index=pd.Index(m.seasonalities.keys(), dtype="object"),
-                    name=s_id,
-                    dtype="object",
+
+            sec3_figs = {
+                s_id: self.models[s_id].plot(self.outputs[s_id]) for s_id in series_ids
+            }
+            for s_id in series_ids:
+                add_changepoints_to_plot(
+                    sec3_figs[s_id].gca(), self.models[s_id], self.outputs[s_id]
                 )
+            sec3 = _select_plot_list(
+                lambda s_id: sec3_figs[s_id], series_ids=series_ids
             )
-        all_model_states = pd.concat(model_states, axis=1)
-        if not all_model_states.empty:
-            sec5 = dp.DataTable(all_model_states)
-            all_sections = all_sections + [sec5_text, sec5]
+            section_3 = rc.Block(rc.Heading("Forecast Changepoints", level=2), sec3)
+
+            all_sections = [section_1, section_2, section_3]
+
+            sec5_text = rc.Heading("Prophet Model Seasonality Components", level=2)
+            model_states = []
+            for s_id in series_ids:
+                m = self.models[s_id]
+                model_states.append(
+                    pd.Series(
+                        m.seasonalities,
+                        index=pd.Index(m.seasonalities.keys(), dtype="object"),
+                        name=s_id,
+                        dtype="object",
+                    )
+                )
+            all_model_states = pd.concat(model_states, axis=1)
+            if not all_model_states.empty:
+                sec5 = rc.DataTable(all_model_states, index=True)
+                all_sections = all_sections + [sec5_text, sec5]
 
         if self.spec.generate_explanations:
             try:
                 # If the key is present, call the "explain_model" method
                 self.explain_model()
 
-                # Create a markdown text block for the global explanation section
-                global_explanation_text = dp.Text(
-                    f"## Global Explanation of Models \n "
-                    "The following tables provide the feature attribution for the global explainability."
-                )
-
                 # Convert the global explanation data to a DataFrame
                 global_explanation_df = pd.DataFrame(self.global_explanation)
 
@@ -313,9 +315,12 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
                 )
 
                 # Create a markdown section for the global explainability
-                global_explanation_section = dp.Blocks(
-                    "### Global Explainability ",
-                    dp.DataTable(self.formatted_global_explanation),
+                global_explanation_section = rc.Block(
+                    rc.Heading("Global Explanation of Models", level=2),
+                    rc.Text(
+                        "The following tables provide the feature attribution for the global explainability."
+                    ),
+                    rc.DataTable(self.formatted_global_explanation, index=True),
                 )
 
                 aggregate_local_explanations = pd.DataFrame()
@@ -327,21 +332,21 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
                     )
                 self.formatted_local_explanation = aggregate_local_explanations
 
-                local_explanation_text = dp.Text(f"## Local Explanation of Models \n ")
                 blocks = [
-                    dp.DataTable(
+                    rc.DataTable(
                         local_ex_df.div(local_ex_df.abs().sum(axis=1), axis=0) * 100,
                         label=s_id,
+                        index=True,
                     )
                     for s_id, local_ex_df in self.local_explanation.items()
                 ]
-                local_explanation_section = (
-                    dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
+                local_explanation_section = rc.Block(
+                    rc.Heading("Local Explanation of Models", level=2),
+                    rc.Select(blocks=blocks),
                 )
 
                 # Append the global explanation text and section to the "all_sections" list
                 all_sections = all_sections + [
-                    global_explanation_text,
                     global_explanation_section,
                     local_explanation_text,
                     local_explanation_section,
@@ -351,7 +356,7 @@ class ProphetOperatorModel(ForecastOperatorBaseModel):
                 logger.warn(f"Failed to generate Explanations with error: {e}.")
                 logger.debug(f"Full Traceback: {traceback.format_exc()}")
 
-        model_description = dp.Text(
+        model_description = (
             "Prophet is a procedure for forecasting time series data based on an additive "
             "model where non-linear trends are fit with yearly, weekly, and daily seasonality, "
             "plus holiday effects. It works best with time series that have strong seasonal "

ads/opctl/operator/lowcode/forecast/operator_config.py

@@ -29,6 +29,22 @@ class DateTimeColumn(DataClassSerializable):
     format: str = None
 
 
+@dataclass(repr=True)
+class PreprocessingSteps(DataClassSerializable):
+    """Class representing preprocessing steps for operator."""
+
+    missing_value_imputation: bool = True
+    outlier_treatment: bool = True
+
+
+@dataclass(repr=True)
+class DataPreprocessor(DataClassSerializable):
+    """Class representing operator specification preprocessing details."""
+
+    enabled: bool = True
+    steps: PreprocessingSteps = field(default_factory=PreprocessingSteps)
+
+
 @dataclass(repr=True)
 class Tuning(DataClassSerializable):
     """Class representing operator specification tuning details."""
@@ -54,7 +70,7 @@ class ForecastOperatorSpec(DataClassSerializable):
     global_explanation_filename: str = None
     local_explanation_filename: str = None
     target_column: str = None
-    preprocessing: bool = None
+    preprocessing: DataPreprocessor = field(default_factory=DataPreprocessor)
     datetime_column: DateTimeColumn = field(default_factory=DateTimeColumn)
     target_category_columns: List[str] = field(default_factory=list)
     generate_report: bool = None
@@ -79,7 +95,7 @@ class ForecastOperatorSpec(DataClassSerializable):
         self.confidence_interval_width = self.confidence_interval_width or 0.80
         self.report_filename = self.report_filename or "report.html"
         self.preprocessing = (
-            self.preprocessing if self.preprocessing is not None else True
+            self.preprocessing if self.preprocessing is not None else DataPreprocessor(enabled=True)
         )
         # For Report Generation. When user doesn't specify defaults to True
         self.generate_report = (

ads/opctl/operator/lowcode/forecast/schema.yaml

@@ -286,11 +286,27 @@ spec:
       default: target
 
     preprocessing:
-      type: boolean
+      type: dict
       required: false
-      default: true
-      meta:
-        description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
+      schema:
+        enabled:
+          type: boolean
+          required: false
+          default: true
+          meta:
+            description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
+        steps:
+          type: dict
+          required: false
+          schema:
+            missing_value_imputation:
+              type: boolean
+              required: false
+              default: false
+            outlier_treatment:
+              type: boolean
+              required: false
+              default: false
 
     generate_explanations:
       type: boolean

ads/opctl/operator/lowcode/forecast/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -51,7 +51,11 @@ def _inverse_transform_dataframe(le, df):
 def smape(actual, predicted) -> float:
     if not all([isinstance(actual, np.ndarray), isinstance(predicted, np.ndarray)]):
         actual, predicted = (np.array(actual), np.array(predicted))
+    zero_mask = np.logical_and(actual == 0, predicted == 0)
+
     denominator = np.abs(actual) + np.abs(predicted)
+    denominator[zero_mask] = 1
+
     numerator = np.abs(actual - predicted)
     default_output = np.ones_like(numerator) * np.inf
 
@@ -254,10 +258,10 @@ def evaluate_train_metrics(output, metrics_col_name=None):
 
 
 def _select_plot_list(fn, series_ids):
-    import datapane as dp
+    import report_creator as rc
 
-    blocks = [dp.Plot(fn(s_id=s_id), label=s_id) for s_id in series_ids]
-    return dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
+    blocks = [rc.Widget(fn(s_id=s_id), label=s_id) for s_id in series_ids]
+    return rc.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
 
 
 def _add_unit(num, unit):

ads/opctl/operator/lowcode/pii/README.md

@@ -33,7 +33,7 @@ To run pii operator locally, create and activate a new conda environment (`ads-p
 
 ```yaml
 - aiohttp
-- datapane
+- report-creator
 - gender_guesser
 - nameparser
 - oracle_ads[opctl]

ads/opctl/operator/lowcode/pii/environment.yaml

@@ -6,7 +6,7 @@ dependencies:
   - pip
   - pip:
       - aiohttp
-      - datapane
+      - report-creator
       - gender_guesser
       - nameparser
       - oracle_ads[opctl]

ads/opctl/operator/lowcode/pii/model/report.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -39,10 +39,10 @@ from ads.opctl.operator.lowcode.pii.operator_config import PiiOperatorConfig
 from ads.opctl.operator.lowcode.pii.utils import compute_rate
 
 try:
-    import datapane as dp
+    import report_creator as rc
 except ImportError:
     raise ModuleNotFoundError(
-        f"`datapane` module was not found. Please run "
+        f"`report-creator` module was not found. Please run "
         f"`pip install {OptionalDependency.PII}`."
     )
 
@@ -119,8 +119,8 @@ def make_model_card(model_name="", readme_path=""):
         logger.warning(
             "You don't have internet connection. Therefore, we are not able to generate model card."
         )
-        return dp.Group(
-            dp.Text("-"),
+        return rc.Group(
+            rc.Text("-"),
             columns=1,
         )
 
@@ -144,15 +144,15 @@ def make_model_card(model_name="", readme_path=""):
                 )
             ]
         )
-        eval_res_tb = dp.Plot(data=fig, caption="Evaluation Results")
+        eval_res_tb = rc.Widget(data=fig, caption="Evaluation Results")
     except:
-        eval_res_tb = dp.Text("-")
+        eval_res_tb = rc.Text("-")
         logger.warning(
             "The given readme.md doesn't have correct template for Evaluation Results."
         )
 
-    return dp.Group(
-        dp.Text(text),
+    return rc.Group(
+        rc.Text(text),
         eval_res_tb,
         columns=2,
     )
@@ -172,7 +172,7 @@ def map_label_to_color(labels):
 
 
 @runtime_dependency(module="plotly", install_from=OptionalDependency.PII)
-def plot_pie(count_map) -> dp.Plot:
+def plot_pie(count_map) -> rc.Widget:
     import plotly.express as px
 
     cols = count_map.keys()
@@ -190,7 +190,7 @@ def plot_pie(count_map) -> dp.Plot:
         color_discrete_map=map_label_to_color(cols),
     )
     fig.update_traces(textposition="inside", textinfo="percent+label")
-    return dp.Plot(fig)
+    return rc.Widget(fig)
 
 
 def build_entity_df(entites, id) -> pd.DataFrame:
@@ -223,37 +223,38 @@ class RowReportFields:
         self.spec = row_spec
         self.show_sensitive_info = show_sensitive_info
 
-    def build_report(self) -> dp.Group:
-        return dp.Group(
-            dp.Select(
+    def build_report(self) -> rc.Group:
+        return rc.Group(
+            rc.Select(
                 blocks=[
                     self._make_stats_card(),
                     self._make_text_card(),
                 ],
-                type=dp.SelectType.TABS,
+                type=rc.SelectType.TABS,
             ),
             label="Row Id: " + str(self.spec.id),
         )
 
     def _make_stats_card(self):
         stats = [
-            dp.Text("## Row Summary Statistics"),
-            dp.BigNumber(
+            rc.Heading("Row Summary Statistics", level=2),
+            rc.Metric(
                 heading="Total No. Of Entites Proceed",
                 value=self.spec.total_tokens or 0,
             ),
-            dp.Text(f"### Entities Distribution"),
+            rc.Heading("Entities Distribution", level=3),
             plot_pie(self.spec.statics),
         ]
         if self.show_sensitive_info:
-            stats.append(dp.Text(f"### Resolved Entities"))
+            stats.append(rc.Heading("Resolved Entities", level=3))
             stats.append(
-                dp.DataTable(
+                rc.DataTable(
                     build_entity_df(self.spec.entities, id=self.spec.id),
                     label="Resolved Entities",
+                    index=True,
                 )
             )
-        return dp.Group(blocks=stats, label="STATS")
+        return rc.Group(stats, label="STATS")
 
     def _make_text_card(self):
         annotations = []
@@ -274,7 +275,7 @@ class RowReportFields:
             },
             return_html=True,
         )
-        return dp.Group(dp.HTML(render_html), label="TEXT")
+        return rc.Group(rc.HTML(render_html), label="TEXT")
 
 
 class PIIOperatorReport:
@@ -293,26 +294,28 @@ class PIIOperatorReport:
         self.report_uri = report_uri
 
     def make_view(self):
-        title_text = dp.Text("# Personally Identifiable Information Operator Report")
-        time_proceed = dp.BigNumber(
+        title_text = rc.Heading(
+            "Personally Identifiable Information Operator Report", level=1
+        )
+        time_proceed = rc.Metric(
             heading="Ran at",
             value=self.report_spec.run_summary.timestamp or "today",
         )
-        report_description = dp.Text(PII_REPORT_DESCRIPTION)
+        report_description = rc.Text(PII_REPORT_DESCRIPTION)
 
-        structure = dp.Blocks(
-            dp.Select(
+        structure = rc.Block(
+            rc.Select(
                 blocks=[
-                    dp.Group(
+                    rc.Group(
                         self._build_summary_page(),
                         label="Summary",
                     ),
-                    dp.Group(
+                    rc.Group(
                         self._build_details_page(),
                         label="Details",
                     ),
                 ],
-                type=dp.SelectType.TABS,
+                type=rc.SelectType.TABS,
             )
         )
         self.report_sections = [title_text, report_description, time_proceed, structure]
@@ -322,11 +325,10 @@ class PIIOperatorReport:
         with tempfile.TemporaryDirectory() as temp_dir:
             report_local_path = os.path.join(temp_dir, "___report.html")
             disable_print()
-            dp.save_report(
-                report_sections or self.report_sections,
-                path=report_local_path,
-                open=False,
-            )
+            with rc.ReportCreator("My Report") as report:
+                report.save(
+                    rc.Block(report_sections or self.report_sections), report_local_path
+                )
             enable_print()
 
             report_uri = report_uri or self.report_uri
@@ -339,36 +341,36 @@ class PIIOperatorReport:
                     f2.write(f1.read())
 
     def _build_summary_page(self):
-        summary = dp.Blocks(
-            dp.Text("# PII Summary"),
-            dp.Text(self._get_summary_desc()),
-            dp.Select(
+        summary = rc.Block(
+            rc.Heading("PII Summary", level=1),
+            rc.Text(self._get_summary_desc()),
+            rc.Select(
                 blocks=[
                     self._make_summary_stats_card(),
                     self._make_yaml_card(),
                     self._make_model_card(),
                 ],
-                type=dp.SelectType.TABS,
+                type=rc.SelectType.TABS,
             ),
         )
 
         return summary
 
     def _build_details_page(self):
-        details = dp.Blocks(
-            dp.Text(DETAILS_REPORT_DESCRIPTION),
-            dp.Select(
+        details = rc.Block(
+            rc.Text(DETAILS_REPORT_DESCRIPTION),
+            rc.Select(
                 blocks=[
                     row.build_report() for row in self.rows_details
                 ],  # RowReportFields
-                type=dp.SelectType.DROPDOWN,
+                type=rc.SelectType.DROPDOWN,
                 label="Details",
             ),
         )
 
         return details
 
-    def _make_summary_stats_card(self) -> dp.Group:
+    def _make_summary_stats_card(self) -> rc.Group:
         """
         Shows summary statics
         1. total rows
@@ -388,21 +390,21 @@ class PIIOperatorReport:
             process_rate = "-"
 
         summary_stats = [
-            dp.Text("## Summary Statistics"),
-            dp.Group(
-                dp.BigNumber(
+            rc.Heading("Summary Statistics", level=2),
+            rc.Group(
+                rc.Metric(
                     heading="Total No. Of Rows",
                     value=self.report_spec.run_summary.total_rows or "unknown",
                 ),
-                dp.BigNumber(
+                rc.Metric(
                     heading="Total No. Of Entites Proceed",
                     value=self.report_spec.run_summary.total_tokens,
                 ),
-                dp.BigNumber(
+                rc.Metric(
                     heading="Rows per second processed",
                     value=process_rate,
                 ),
-                dp.BigNumber(
+                rc.Metric(
                     heading="Total Time Spent",
                     value=human_time_friendly(
                         self.report_spec.run_summary.elapsed_time
@@ -410,32 +412,31 @@ class PIIOperatorReport:
                 ),
                 columns=2,
             ),
-            dp.Text(f"### Entities Distribution"),
+            rc.Heading("Entities Distribution", level=3),
             plot_pie(self.report_spec.run_summary.statics),
         ]
         if self.report_spec.run_summary.show_sensitive_info:
             entites_df = self._build_total_entity_df()
-            summary_stats.append(dp.Text(f"### Resolved Entities"))
-            summary_stats.append(dp.DataTable(entites_df))
-        return dp.Group(blocks=summary_stats, label="STATS")
+            summary_stats.append(rc.Heading("Resolved Entities", level=3))
+            summary_stats.append(rc.DataTable(entites_df, index=True))
+        return rc.Group(summary_stats, label="STATS")
 
-    def _make_yaml_card(self) -> dp.Group:
+    def _make_yaml_card(self) -> rc.Group:
         """Shows the full pii config yaml."""
-        yaml_string = self.report_spec.run_summary.config.to_yaml()
-        yaml_appendix_title = dp.Text(f"## Reference: YAML File")
-        yaml_appendix = dp.Code(code=yaml_string, language="yaml")
-        return dp.Group(blocks=[yaml_appendix_title, yaml_appendix], label="YAML")
+        yaml_appendix_title = rc.Heading("Reference: YAML File", level=2)
+        yaml_appendix = rc.Yaml(self.report_spec.run_summary.config.to_dict())
+        return rc.Group(yaml_appendix_title, yaml_appendix, label="YAML")
 
-    def _make_model_card(self) -> dp.Group:
+    def _make_model_card(self) -> rc.Group:
         """Generates model card."""
         if len(self.report_spec.run_summary.selected_spacy_model) == 0:
-            return dp.Group(
-                dp.Text("No model used."),
+            return rc.Group(
+                rc.Text("No model used."),
                 label="MODEL CARD",
             )
 
         model_cards = [
-            dp.Group(
+            rc.Group(
                 make_model_card(model_name=x.get("model")),
                 label=x.get("model"),
             )
@@ -443,14 +444,14 @@ class PIIOperatorReport:
         ]
 
         if len(model_cards) <= 1:
-            return dp.Group(
-                blocks=model_cards,
+            return rc.Group(
+                model_cards,
                 label="MODEL CARD",
             )
-        return dp.Group(
-            dp.Select(
-                blocks=model_cards,
-                type=dp.SelectType.TABS,
+        return rc.Group(
+            rc.Select(
+                model_cards,
+                type=rc.SelectType.TABS,
             ),
             label="MODEL CARD",
         )