analysistoolbox 2.2.0__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/PKG-INFO +1 -1
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateMetalogDistribution.py +46 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/__init__.py +3 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/a_vector.py +341 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/class_method.py +462 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/metalog.py +485 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/pdf_quantile_functions.py +117 -0
- analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/support.py +390 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/PKG-INFO +1 -1
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/SOURCES.txt +6 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/setup.py +1 -1
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/LICENSE.md +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/README.md +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindDerivative.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindLimitOfFunction.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindMinimumSquareLoss.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/PlotFunction.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/ExtractTextFromPDF.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchPDFFromURL.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchUSShapefile.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchWebsiteText.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetCompanyFilings.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetGoogleSearchResults.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetZipFile.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddDateNumberColumns.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddLeadingZeros.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddRowCountColumn.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddTPeriodColumn.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddTukeyOutlierColumn.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CleanTextColumns.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConductAnomalyDetection.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConductEntityMatching.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConvertOddsToProbability.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CountMissingDataByGroup.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateBinnedColumn.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateDataOverview.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateRandomSampleGroups.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateRareCategoryColumn.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateStratifiedRandomSampleGroups.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/GeocodeUSAddresses.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ImputeMissingValuesUsingNearestNeighbors.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/VerifyGranularity.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductManifoldLearning.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductPrincipalComponentAnalysis.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductPropensityScoreMatching.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateAssociationRules.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateGaussianMixtureClusters.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateHierarchicalClusters.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateKMeansClusters.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/GenerateEDAWithLIDA.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/ConvertWordDocsToPDF.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/CreateCopyOfPDF.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/CreateFileTree.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/ImportDataFromFolder.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ChiSquareTestOfIndependence.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ChiSquareTestOfIndependenceFromTable.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductCoxProportionalHazardRegression.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductLinearRegressionAnalysis.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductLogisticRegressionAnalysis.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductSurvivalAnalysis.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/OneSampleTTest.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/OneWayANOVA.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfMeanFromStats.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfProportionFromStats.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfTwoMeansFromStats.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TwoSampleTTestOfIndependence.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TwoSampleTTestPaired.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/CalculateEigenvalues.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/ConvertMatrixToRowEchelonForm.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/ConvertSystemOfEquationsToMatrix.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/PlotVectors.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/SolveSystemOfEquations.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/VisualizeMatrixAsLinearTransformation.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/SendPromptToAnthropic.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/SendPromptToChatGPT.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateARIMAModel.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateBoostedTreeModel.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateDecisionTreeModel.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateLinearRegressionModel.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateLogisticRegressionModel.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateNeuralNetwork_SingleOutcome.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/prescriptive_analytics/CreateContentBasedRecommender.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/prescriptive_analytics/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/probability/ProbabilityOfAtLeastOne.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/probability/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateMetalogDistributionFromPercentiles.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateSIPDataframe.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateSLURPDistribution.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountOfSuccesses.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountOutcome.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountUntilFirstSuccess.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateNormallyDistributedOutcome.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTDistributedOutcome.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTimeBetweenEvents.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTimeUntilNEvents.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/CalculateConfidenceIntervalOfMean.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/CalculateConfidenceIntervalOfProportion.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/Plot100PercentStackedBarChart.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBarChart.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBoxWhiskerByGroup.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBulletChart.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotCard.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotClusteredBarChart.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotContingencyHeatmap.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotCorrelationMatrix.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotDensityByGroup.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotDotPlot.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotHeatmap.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotOverlappingAreaChart.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotRiskTolerance.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotScatterplot.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotSingleVariableCountPlot.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotSingleVariableHistogram.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotTimeSeries.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/RenderTableOne.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/__init__.py +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/dependency_links.txt +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/requires.txt +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/top_level.txt +0 -0
- {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: analysistoolbox
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.1
|
|
4
4
|
Summary: A collection tools in Python for data collection and processing, statistics, analytics, and intelligence analysis.
|
|
5
5
|
Home-page: https://github.com/KyleProtho/AnalysisToolBox/tree/master/analysistoolbox
|
|
6
6
|
Author: Kyle Protho
|
|
@@ -87,7 +87,22 @@ def CreateMetalogDistribution(dataframe,
|
|
|
87
87
|
# Extract values from the dataframe
|
|
88
88
|
arr_variable = dataframe[variable]
|
|
89
89
|
arr_variable = list(arr_variable)
|
|
90
|
+
|
|
91
|
+
# If there is a lower_bound, print a warning and filter the data
|
|
92
|
+
if lower_bound is not None:
|
|
93
|
+
if min(arr_variable) < lower_bound:
|
|
94
|
+
print(f"Warning: There are observations that are lower than the lower bound of {lower_bound}. Filtering data...")
|
|
95
|
+
arr_variable = [x for x in arr_variable if x >= lower_bound]
|
|
96
|
+
|
|
97
|
+
# If there is an upper_bound, print a warning and filter the data
|
|
98
|
+
if upper_bound is not None:
|
|
99
|
+
if max(arr_variable) > upper_bound:
|
|
100
|
+
print(f"Warning: There are observations that are higher than the upper bound of {upper_bound}. Filtering data...")
|
|
101
|
+
arr_variable = [x for x in arr_variable if x <= upper_bound]
|
|
90
102
|
|
|
103
|
+
# Turn off warnings
|
|
104
|
+
import warnings
|
|
105
|
+
warnings.filterwarnings("ignore")
|
|
91
106
|
# Create a metalog distribution
|
|
92
107
|
if lower_bound is None and upper_bound is None:
|
|
93
108
|
metalog_dist = pm.metalog(
|
|
@@ -294,3 +309,34 @@ def CreateMetalogDistribution(dataframe,
|
|
|
294
309
|
else:
|
|
295
310
|
return arr_metalog
|
|
296
311
|
|
|
312
|
+
|
|
313
|
+
# Test the function using upper and lower bounds
|
|
314
|
+
from sklearn.datasets import load_iris
|
|
315
|
+
iris_df = pd.DataFrame(load_iris().data, columns=load_iris().feature_names)
|
|
316
|
+
CreateMetalogDistribution(
|
|
317
|
+
dataframe=iris_df,
|
|
318
|
+
variable='sepal length (cm)',
|
|
319
|
+
lower_bound=2,
|
|
320
|
+
upper_bound=7,
|
|
321
|
+
learning_rate=0.01,
|
|
322
|
+
term_maximum=9,
|
|
323
|
+
term_minimum=2,
|
|
324
|
+
term_for_random_sample=None,
|
|
325
|
+
number_of_samples=10000,
|
|
326
|
+
show_summary=True,
|
|
327
|
+
return_format='dataframe',
|
|
328
|
+
plot_metalog_distribution=True,
|
|
329
|
+
fill_color="#999999",
|
|
330
|
+
fill_transparency=0.6,
|
|
331
|
+
figure_size=(8, 6),
|
|
332
|
+
show_mean=True,
|
|
333
|
+
show_median=True,
|
|
334
|
+
title_for_plot="Metalog Distribution",
|
|
335
|
+
subtitle_for_plot="Showing the metalog distribution of the variable",
|
|
336
|
+
caption_for_plot=None,
|
|
337
|
+
data_source_for_plot=None,
|
|
338
|
+
show_y_axis=False,
|
|
339
|
+
title_y_indent=1.1,
|
|
340
|
+
subtitle_y_indent=1.05,
|
|
341
|
+
caption_y_indent=-0.15
|
|
342
|
+
)
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import numpy as np
|
|
3
|
+
import scipy as sp
|
|
4
|
+
|
|
5
|
+
from scipy.optimize import linprog, minimize, NonlinearConstraint
|
|
6
|
+
from .pdf_quantile_functions import pdf_quantile_builder
|
|
7
|
+
from .support import diffMatMetalog, pdfMetalog, quantileMetalog, newtons_method_metalog
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
import warnings
|
|
11
|
+
|
|
12
|
+
def a_vector_OLS_and_LP(m_dict,
|
|
13
|
+
bounds,
|
|
14
|
+
boundedness,
|
|
15
|
+
term_limit,
|
|
16
|
+
term_lower_bound,
|
|
17
|
+
fit_method,
|
|
18
|
+
alpha,
|
|
19
|
+
diff_error = .001,
|
|
20
|
+
diff_step = 0.001):
|
|
21
|
+
|
|
22
|
+
""" Main workhorse function of pymetalog package.
|
|
23
|
+
Called during metalog.__init__ method call.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
m_dict (:obj:`dict` with keys ['params', 'dataValues', 'Y']): Initialized output_dict variable from metalog class.
|
|
27
|
+
- m_dict['params']: (:obj:`dict` with keys ['bounds', 'boundedness', 'term_limit', 'term_lower_bound', 'step_len', 'fit_method']):
|
|
28
|
+
* 'bounds': metalog.bounds
|
|
29
|
+
* 'boundedness': metalog.boundedness
|
|
30
|
+
* 'term_limit': metalog.term_limit
|
|
31
|
+
* 'term_lower_bound': metalog.term_lower_bound
|
|
32
|
+
* 'step_len': metalog.step_len
|
|
33
|
+
* 'fit_method': metalog.fit_method
|
|
34
|
+
|
|
35
|
+
- m_dict['dataValues']: (:obj:`pandas.DataFrame` with columns ['x','probs','z'] of type numeric):
|
|
36
|
+
* 'x': metalog.x
|
|
37
|
+
* 'probs': metalog.probs
|
|
38
|
+
* 'z': column calculated in metalog.append_zvector method
|
|
39
|
+
- depends on metalog.boundedness attribute
|
|
40
|
+
- metalog.boundedness = 'u':
|
|
41
|
+
* 'z' = metalog.x
|
|
42
|
+
- metalog.boundedness = 'sl':
|
|
43
|
+
* 'z' = log( (metalog.x-lower_bound) )
|
|
44
|
+
- metalog.boundedness = 'su':
|
|
45
|
+
* 'z' = = log( (upper_bound-metalog.x) )
|
|
46
|
+
- metalog.boundedness = 'b':
|
|
47
|
+
* 'z' = log( (metalog.x-lower_bound) / (upper_bound-metalog.x) )
|
|
48
|
+
|
|
49
|
+
- m_dict['Y']: (:obj:`pandas.DataFrame` with columns ['y1','y2','y3','y4', ... ,'yn'] of type numeric):
|
|
50
|
+
* 'y1': numpy.array of ones with length equal to len(x)
|
|
51
|
+
* 'y2': numpy.array of numeric values equal to the term attached to s in the logistic quantile function np.log(m_dict['dataValues']['probs'] / (1 - m_dict['dataValues']['probs']))
|
|
52
|
+
* 'y3': numpy.array of numeric values (m_dict['dataValues']['probs'] - 0.5) * m_dict['Y']['y2']
|
|
53
|
+
* 'y4': numpy.array of numeric values m_dict['Y']['y4'] = m_dict['dataValues']['probs'] - 0.5
|
|
54
|
+
* 'yn': numpy.array of numeric values:
|
|
55
|
+
- if n in 'yn' is odd,
|
|
56
|
+
m_dict['Y']['yn'] = m_dict['Y']['y4']**(int(i//2))
|
|
57
|
+
- if n in 'yn' is even,
|
|
58
|
+
zn = 'y' + str(n-1)
|
|
59
|
+
m_dict['Y'][yn] = m_dict['Y']['y2'] * m_dict['Y'][zn]
|
|
60
|
+
|
|
61
|
+
bounds (:obj:`list`): Upper and lower limits to filter the data with before calculating metalog quantiles/pdfs.
|
|
62
|
+
- should be set in conjunction with the `boundedness` parameter
|
|
63
|
+
|
|
64
|
+
boundedness (:obj:`str`): String that is used to specify the type of metalog to fit.
|
|
65
|
+
- must be in set ('u','sl','su','b')
|
|
66
|
+
- Default: 'u'
|
|
67
|
+
* Fits an unbounded metalog
|
|
68
|
+
- 'sl' fits a strictly lower bounded metalog
|
|
69
|
+
* len(bounds) must == 1
|
|
70
|
+
- 'su' fits a strictly upper bounded metalog
|
|
71
|
+
* len(bounds) must == 1
|
|
72
|
+
- 'b' fits a upper/lower bounded metalog
|
|
73
|
+
* len(bounds) must == 2
|
|
74
|
+
* bounds[1] must be > bounds[0]
|
|
75
|
+
|
|
76
|
+
term_limit (:obj:`int`): The upper limit of the range of metalog terms to use to fit the data.
|
|
77
|
+
- strictly > term_lower_bound
|
|
78
|
+
- in range [3,30]
|
|
79
|
+
|
|
80
|
+
term_lower_bound (:obj:`int`): The lower limit of the range of metalog terms to use to fit the data.
|
|
81
|
+
- strictly < term_limit
|
|
82
|
+
- in range [2,29]
|
|
83
|
+
|
|
84
|
+
fit_method (:obj:`str`): Fit method to use to fit metalog distribution.
|
|
85
|
+
- must be in set ('any','OLS','LP','MLE')
|
|
86
|
+
- Default: 'any'
|
|
87
|
+
* first tries 'OLS' method than 'LP'
|
|
88
|
+
- 'OLS' only tries to fit by solving directly for a coefficients using ordinary least squares method
|
|
89
|
+
- 'LP' only tries to estimate fit using simplex linear program optimization routine
|
|
90
|
+
- 'MLE' first tries 'OLS' method than falls back to a maximum likelihood estimation routine
|
|
91
|
+
|
|
92
|
+
alpha (:obj:`float`, optional): Regularization term to add to OLS fit
|
|
93
|
+
- strictly >= 0.
|
|
94
|
+
- should be set in conjunction with `penalty` parameter
|
|
95
|
+
- Default: 0. (no regularization, OLS)
|
|
96
|
+
|
|
97
|
+
diff_error (:obj:`float`, optional): Value used to in scipy.optimize.linprog method call
|
|
98
|
+
to init the array of values representing the
|
|
99
|
+
upper-bound of each inequality constraint (row) in A_ub.
|
|
100
|
+
- #TODO: Insert maths
|
|
101
|
+
|
|
102
|
+
diff_step (:obj:`float`, optional): Value passed to `step_len` parameter in support.py diffMatMetalog method call
|
|
103
|
+
defines the bin width for the Reimann sum of the differences differentiation method
|
|
104
|
+
- diffMatMetalog differentiates the metalog pdf
|
|
105
|
+
* Differentiation reference: https://math.stackexchange.com/a/313135
|
|
106
|
+
Returns:
|
|
107
|
+
m_dict: (:obj:`dict` with keys ['params', 'dataValues', 'Y', 'A', 'M', 'Validation'])
|
|
108
|
+
- m_dict['A']: (:obj:`pandas.DataFrame` with columns ['a2','a3', ... ,'an'] of type numeric):
|
|
109
|
+
* a2, a3, ... , an are our a coefficients returned by the method specified in `fit_method`
|
|
110
|
+
|
|
111
|
+
- m_dict['M']: (:obj:`pandas.DataFrame` with columns 0:'pdf_1',1:'cdf_1',2:'pdf_2',3:'cdf_2',
|
|
112
|
+
...,((2*(term_limit-term_lower_bound))+1)-1:'pdf_n',
|
|
113
|
+
((2*(term_limit-term_lower_bound))+1):'cdf_n'
|
|
114
|
+
where n is the total number of metalog fits determined by (term_limit-term_lower_bound)+1
|
|
115
|
+
)
|
|
116
|
+
* pdf_1, pdf_2, ... , pdf_n are the metalog pdfs returned by pdf_quantile_builder.pdfMetalog method
|
|
117
|
+
* cdf_1, cdf_2, ... , cdf_n are the metalog quantiles returned by pdf_quantile_builder.quantileMetalog method
|
|
118
|
+
|
|
119
|
+
- m_dict['y']: (:obj: `numpy.ndarray` of type float):
|
|
120
|
+
* Array of bin widths for both the pdf_n and cdf_n
|
|
121
|
+
|
|
122
|
+
- m_dict['Validation']: (:obj:`pandas.DataFrame` with columns ['term', 'valid', 'method'] of type str):
|
|
123
|
+
* 'term': each metalog estimation given a number of terms
|
|
124
|
+
* 'valid': boolean flag indicating if the metalog estimation was valid or not
|
|
125
|
+
* 'method': a string indicating which method was used for the metalog estimation
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
A = pd.DataFrame()
|
|
130
|
+
c_a_names = []
|
|
131
|
+
c_m_names = []
|
|
132
|
+
Mh = pd.DataFrame()
|
|
133
|
+
Validation = pd.DataFrame()
|
|
134
|
+
df_MH_temp_list = list()
|
|
135
|
+
df_A_temp_list = list()
|
|
136
|
+
df_Validation_temp_list = list()
|
|
137
|
+
|
|
138
|
+
# TODO: Large for-loop can probably be factored into smaller functions
|
|
139
|
+
for i in range(term_lower_bound,term_limit+1):
|
|
140
|
+
Y = m_dict['Y'].iloc[:,0:i]
|
|
141
|
+
eye = np.eye(Y.shape[1])
|
|
142
|
+
z = m_dict['dataValues']['z']
|
|
143
|
+
y = m_dict['dataValues']['probs']
|
|
144
|
+
step_len = m_dict['params']['step_len']
|
|
145
|
+
methodFit = 'OLS'
|
|
146
|
+
a_name = 'a'+str(i)
|
|
147
|
+
m_name = 'm'+str(i)
|
|
148
|
+
M_name = 'M'+str(i)
|
|
149
|
+
c_m_names = np.append(c_m_names, [m_name, M_name])
|
|
150
|
+
c_a_names = np.append(c_a_names, a_name)
|
|
151
|
+
|
|
152
|
+
if fit_method == 'any' or fit_method == 'MLE':
|
|
153
|
+
try:
|
|
154
|
+
temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
|
|
155
|
+
except:
|
|
156
|
+
# use LP solver if OLS breaks
|
|
157
|
+
temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
|
|
158
|
+
methodFit = 'Linear Program'
|
|
159
|
+
if fit_method == 'OLS':
|
|
160
|
+
try:
|
|
161
|
+
temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
|
|
162
|
+
except:
|
|
163
|
+
raise RuntimeError("OLS was unable to solve infeasible or poorly formulated problem")
|
|
164
|
+
if fit_method == "LP":
|
|
165
|
+
temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
|
|
166
|
+
methodFit = 'Linear Program'
|
|
167
|
+
|
|
168
|
+
if fit_method == 'MLE':
|
|
169
|
+
temp = a_vector_MLE(temp, y, i, m_dict, bounds, boundedness)
|
|
170
|
+
|
|
171
|
+
temp = np.append(temp, np.zeros(term_limit-i))
|
|
172
|
+
|
|
173
|
+
# build a y vector for smaller data sets
|
|
174
|
+
if len(z) < 100:
|
|
175
|
+
y2 = np.linspace(step_len, 1 - step_len, int((1 - step_len) / step_len))
|
|
176
|
+
tailstep = step_len / 10
|
|
177
|
+
y1 = np.linspace(tailstep, (min(y2) - tailstep), int((min(y2) - tailstep) / tailstep))
|
|
178
|
+
y3 = np.linspace((max(y2) + tailstep), (max(y2) + tailstep * 9), int((tailstep * 9) / tailstep))
|
|
179
|
+
y = np.hstack((y1, y2, y3))
|
|
180
|
+
|
|
181
|
+
# Get the dict and quantile values back for validation
|
|
182
|
+
temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)
|
|
183
|
+
|
|
184
|
+
# If it not a valid pdf run and the OLS version was used the LP version
|
|
185
|
+
if (temp_dict['valid'] == 'no') and (fit_method != 'OLS'):
|
|
186
|
+
temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
|
|
187
|
+
temp = np.append(temp, np.zeros(term_limit-i))
|
|
188
|
+
methodFit = 'Linear Program'
|
|
189
|
+
|
|
190
|
+
# Get the dict and quantile values back for validation
|
|
191
|
+
temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)
|
|
192
|
+
|
|
193
|
+
df_MH_temp_list.append(pd.DataFrame(temp_dict['m']))
|
|
194
|
+
df_MH_temp_list.append(pd.DataFrame(temp_dict['M']))
|
|
195
|
+
df_A_temp_list.append(pd.DataFrame(temp))
|
|
196
|
+
|
|
197
|
+
tempValidation = pd.DataFrame(data={'term': [i], 'valid': [temp_dict['valid']], 'method': [methodFit]})
|
|
198
|
+
df_Validation_temp_list.append(tempValidation)
|
|
199
|
+
|
|
200
|
+
Validation = pd.concat(df_Validation_temp_list, axis=0)
|
|
201
|
+
Mh = pd.concat(df_MH_temp_list, axis=1)
|
|
202
|
+
A = pd.concat(df_A_temp_list, axis=1)
|
|
203
|
+
|
|
204
|
+
A.columns = c_a_names
|
|
205
|
+
Mh.columns = c_m_names
|
|
206
|
+
|
|
207
|
+
m_dict['A'] = A
|
|
208
|
+
m_dict['M'] = Mh
|
|
209
|
+
m_dict['M']['y'] = temp_dict['y']
|
|
210
|
+
m_dict['Validation'] = Validation
|
|
211
|
+
|
|
212
|
+
A = np.column_stack((np.repeat(1.,len(A)), A))
|
|
213
|
+
Est = np.dot(m_dict['Y'], A)
|
|
214
|
+
ncols = A.shape[1]
|
|
215
|
+
try:
|
|
216
|
+
Z = np.column_stack((np.array(m_dict['dataValues']['z']),np.repeat(m_dict['dataValues']['z'].values,ncols-1).reshape(len(m_dict['dataValues']['z']),ncols-1)))
|
|
217
|
+
except:
|
|
218
|
+
Z = np.column_stack((np.array(m_dict['dataValues']['z']),np.repeat(m_dict['dataValues']['z'], ncols-1).reshape(len(m_dict['dataValues']['z']),ncols-1)))
|
|
219
|
+
|
|
220
|
+
m_dict['square_residual_error'] = ((Z-Est)**2).sum(axis=1)
|
|
221
|
+
|
|
222
|
+
return m_dict
|
|
223
|
+
|
|
224
|
+
def a_vector_LP(m_dict, term_limit, term_lower_bound, diff_error = .001, diff_step = 0.001):
|
|
225
|
+
"""TODO: write docstring
|
|
226
|
+
|
|
227
|
+
"""
|
|
228
|
+
cnames = np.array([])
|
|
229
|
+
|
|
230
|
+
for i in range(term_lower_bound, term_limit + 1):
|
|
231
|
+
Y = m_dict['Y'].iloc[:, 0:i]
|
|
232
|
+
z = m_dict['dataValues']['z']
|
|
233
|
+
|
|
234
|
+
# Bulding the objective function using abs value LP formulation
|
|
235
|
+
Y_neg = -Y
|
|
236
|
+
|
|
237
|
+
new_Y = pd.DataFrame({'y1': Y.iloc[:, 0], 'y1_neg': Y_neg.iloc[:, 0]})
|
|
238
|
+
|
|
239
|
+
for c in range(1,len(Y.iloc[0,:])):
|
|
240
|
+
new_Y['y'+str(c+1)] = Y.iloc[:,c]
|
|
241
|
+
new_Y['y' + str(c+1)+'_neg'] = Y_neg.iloc[:, c]
|
|
242
|
+
|
|
243
|
+
a = np.array([''.join(['a', str(i)])])
|
|
244
|
+
cnames = np.append(cnames, a, axis=0)
|
|
245
|
+
|
|
246
|
+
# Building the constraint matrix
|
|
247
|
+
error_mat = np.array([])
|
|
248
|
+
|
|
249
|
+
for j in range(1,len(Y.iloc[:,0])+1):
|
|
250
|
+
front_zeros = np.zeros(2 * (j - 1))
|
|
251
|
+
ones = [1, -1]
|
|
252
|
+
trail_zeroes = np.zeros(2 * (len(Y.iloc[:, 1]) - j))
|
|
253
|
+
if j == 1:
|
|
254
|
+
error_vars = np.append(ones, trail_zeroes)
|
|
255
|
+
|
|
256
|
+
elif j != 1:
|
|
257
|
+
error_vars = np.append(front_zeros, ones)
|
|
258
|
+
error_vars = np.append(error_vars, trail_zeroes)
|
|
259
|
+
|
|
260
|
+
if error_mat.size == 0:
|
|
261
|
+
error_mat = np.append(error_mat, error_vars, axis=0)
|
|
262
|
+
else:
|
|
263
|
+
error_mat = np.vstack((error_mat, error_vars))
|
|
264
|
+
|
|
265
|
+
new = pd.concat((pd.DataFrame(data=error_mat), new_Y), axis=1)
|
|
266
|
+
diff_mat = diffMatMetalog(i, diff_step)
|
|
267
|
+
diff_zeros = []
|
|
268
|
+
|
|
269
|
+
for t in range(0,len(diff_mat.iloc[:, 0])):
|
|
270
|
+
zeros_temp = np.zeros(2 * len(Y.iloc[:, 0]))
|
|
271
|
+
|
|
272
|
+
if np.size(diff_zeros) == 0:
|
|
273
|
+
diff_zeros = zeros_temp
|
|
274
|
+
else:
|
|
275
|
+
diff_zeros = np.vstack((zeros_temp, diff_zeros))
|
|
276
|
+
|
|
277
|
+
diff_mat = np.concatenate((diff_zeros, diff_mat), axis=1)
|
|
278
|
+
|
|
279
|
+
# Combine the total constraint matrix
|
|
280
|
+
lp_mat = np.concatenate((new, diff_mat), axis=0)
|
|
281
|
+
|
|
282
|
+
# Objective function coeficients
|
|
283
|
+
c = np.append(np.ones(2 * len(Y.iloc[:, 1])), np.zeros(2*i))
|
|
284
|
+
|
|
285
|
+
# Constraint matrices
|
|
286
|
+
A_eq = lp_mat[:len(Y.iloc[:, 1]),:]
|
|
287
|
+
A_ub = -1*lp_mat[len(Y.iloc[:, 1]):,:]
|
|
288
|
+
b_eq = z
|
|
289
|
+
b_ub = -1*np.repeat(diff_error, len(diff_mat[:,0]))
|
|
290
|
+
|
|
291
|
+
# Solving the linear program w/ scipy (for now)
|
|
292
|
+
lp_sol = linprog(c, A_ub=A_ub, b_ub=b_ub, A_eq=A_eq, b_eq=b_eq, method='simplex', options={"maxiter":5000, "tol":1.0e-5,"disp": False})
|
|
293
|
+
|
|
294
|
+
# Consolidating solution back into the a vector
|
|
295
|
+
tempLP = lp_sol.x[(2 * len(Y.iloc[:, 1])):(len(lp_sol.x)+1)]
|
|
296
|
+
temp = []
|
|
297
|
+
|
|
298
|
+
for r in range(0,((len(tempLP) // 2))):
|
|
299
|
+
temp.append(tempLP[(r * 2)] - tempLP[(2 * r)+1])
|
|
300
|
+
|
|
301
|
+
return temp
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def a_vector_MLE(a, y, term, m_dict, bounds, boundedness):
|
|
305
|
+
"""TODO: write docstring
|
|
306
|
+
|
|
307
|
+
"""
|
|
308
|
+
ym = [newtons_method_metalog(a, xi, term, bounds, boundedness) for xi in m_dict['dataValues']['x']]
|
|
309
|
+
|
|
310
|
+
def MLE_quantile_constraints(x):
|
|
311
|
+
M = [quantileMetalog(x[:term], yi, term, bounds=bounds, boundedness=boundedness) for yi in x[term:]]
|
|
312
|
+
return m_dict['dataValues']['x'] - M
|
|
313
|
+
|
|
314
|
+
def MLE_objective_function(x, y, term, m_dict):
|
|
315
|
+
return -np.sum([np.log10(pdfMetalog(x[:term], yi, term, bounds, boundedness)) for yi in np.absolute(x[term:])])
|
|
316
|
+
|
|
317
|
+
m_dict[str('MLE' + str(term))] = {}
|
|
318
|
+
|
|
319
|
+
x0 = np.hstack((a[:term],ym))
|
|
320
|
+
m_dict[str('MLE' + str(term))]['oldobj'] = -MLE_objective_function(x0, y, term, m_dict)
|
|
321
|
+
bnd = ((None, None),)*len(a)+((0, 1),)*(len(x0)-len(a))
|
|
322
|
+
con = NonlinearConstraint(MLE_quantile_constraints, 0, 0)
|
|
323
|
+
|
|
324
|
+
mle = minimize(MLE_objective_function, x0, args=(y, term, m_dict), bounds=bnd, constraints=con)
|
|
325
|
+
|
|
326
|
+
m_dict[str('MLE' + str(term))]['newobj'] = -MLE_objective_function(mle.x, y, term, m_dict)
|
|
327
|
+
m_dict[str('MLE'+str(term))]['A'] = mle.x[:term]
|
|
328
|
+
m_dict[str('MLE'+str(term))]['Y'] = mle.x[term:]
|
|
329
|
+
|
|
330
|
+
m_dict[str('MLE' + str(term))]['oldA'] = a
|
|
331
|
+
m_dict[str('MLE' + str(term))]['oldY'] = y
|
|
332
|
+
|
|
333
|
+
out_temp = np.zeros_like(a)
|
|
334
|
+
for i in range(term):
|
|
335
|
+
out_temp[i] = mle.x[i]
|
|
336
|
+
|
|
337
|
+
return out_temp
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
|