analysistoolbox 2.2.0__tar.gz → 2.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/PKG-INFO +1 -1
  2. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateMetalogDistribution.py +46 -0
  3. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/__init__.py +3 -0
  4. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/a_vector.py +341 -0
  5. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/class_method.py +462 -0
  6. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/metalog.py +485 -0
  7. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/pdf_quantile_functions.py +117 -0
  8. analysistoolbox-2.2.1/analysistoolbox/simulations/pymetalog/support.py +390 -0
  9. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/PKG-INFO +1 -1
  10. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/SOURCES.txt +6 -0
  11. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/setup.py +1 -1
  12. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/LICENSE.md +0 -0
  13. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/README.md +0 -0
  14. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/__init__.py +0 -0
  15. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindDerivative.py +0 -0
  16. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindLimitOfFunction.py +0 -0
  17. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/FindMinimumSquareLoss.py +0 -0
  18. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/PlotFunction.py +0 -0
  19. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/calculus/__init__.py +0 -0
  20. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/ExtractTextFromPDF.py +0 -0
  21. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchPDFFromURL.py +0 -0
  22. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchUSShapefile.py +0 -0
  23. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/FetchWebsiteText.py +0 -0
  24. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetCompanyFilings.py +0 -0
  25. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetGoogleSearchResults.py +0 -0
  26. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/GetZipFile.py +0 -0
  27. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_collection/__init__.py +0 -0
  28. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddDateNumberColumns.py +0 -0
  29. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddLeadingZeros.py +0 -0
  30. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddRowCountColumn.py +0 -0
  31. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddTPeriodColumn.py +0 -0
  32. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/AddTukeyOutlierColumn.py +0 -0
  33. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CleanTextColumns.py +0 -0
  34. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConductAnomalyDetection.py +0 -0
  35. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConductEntityMatching.py +0 -0
  36. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ConvertOddsToProbability.py +0 -0
  37. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CountMissingDataByGroup.py +0 -0
  38. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateBinnedColumn.py +0 -0
  39. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateDataOverview.py +0 -0
  40. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateRandomSampleGroups.py +0 -0
  41. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateRareCategoryColumn.py +0 -0
  42. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/CreateStratifiedRandomSampleGroups.py +0 -0
  43. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/GeocodeUSAddresses.py +0 -0
  44. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/ImputeMissingValuesUsingNearestNeighbors.py +0 -0
  45. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/VerifyGranularity.py +0 -0
  46. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/data_processing/__init__.py +0 -0
  47. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductManifoldLearning.py +0 -0
  48. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductPrincipalComponentAnalysis.py +0 -0
  49. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/ConductPropensityScoreMatching.py +0 -0
  50. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateAssociationRules.py +0 -0
  51. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateGaussianMixtureClusters.py +0 -0
  52. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateHierarchicalClusters.py +0 -0
  53. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/CreateKMeansClusters.py +0 -0
  54. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/GenerateEDAWithLIDA.py +0 -0
  55. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/descriptive_analytics/__init__.py +0 -0
  56. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/ConvertWordDocsToPDF.py +0 -0
  57. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/CreateCopyOfPDF.py +0 -0
  58. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/CreateFileTree.py +0 -0
  59. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/ImportDataFromFolder.py +0 -0
  60. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/file_management/__init__.py +0 -0
  61. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ChiSquareTestOfIndependence.py +0 -0
  62. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ChiSquareTestOfIndependenceFromTable.py +0 -0
  63. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductCoxProportionalHazardRegression.py +0 -0
  64. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductLinearRegressionAnalysis.py +0 -0
  65. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductLogisticRegressionAnalysis.py +0 -0
  66. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/ConductSurvivalAnalysis.py +0 -0
  67. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/OneSampleTTest.py +0 -0
  68. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/OneWayANOVA.py +0 -0
  69. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfMeanFromStats.py +0 -0
  70. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfProportionFromStats.py +0 -0
  71. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TTestOfTwoMeansFromStats.py +0 -0
  72. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TwoSampleTTestOfIndependence.py +0 -0
  73. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/TwoSampleTTestPaired.py +0 -0
  74. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/hypothesis_testing/__init__.py +0 -0
  75. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/CalculateEigenvalues.py +0 -0
  76. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/ConvertMatrixToRowEchelonForm.py +0 -0
  77. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/ConvertSystemOfEquationsToMatrix.py +0 -0
  78. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/PlotVectors.py +0 -0
  79. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/SolveSystemOfEquations.py +0 -0
  80. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/VisualizeMatrixAsLinearTransformation.py +0 -0
  81. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/linear_algebra/__init__.py +0 -0
  82. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/SendPromptToAnthropic.py +0 -0
  83. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/SendPromptToChatGPT.py +0 -0
  84. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/llm/__init__.py +0 -0
  85. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateARIMAModel.py +0 -0
  86. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateBoostedTreeModel.py +0 -0
  87. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateDecisionTreeModel.py +0 -0
  88. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateLinearRegressionModel.py +0 -0
  89. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateLogisticRegressionModel.py +0 -0
  90. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/CreateNeuralNetwork_SingleOutcome.py +0 -0
  91. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/predictive_analytics/__init__.py +0 -0
  92. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/prescriptive_analytics/CreateContentBasedRecommender.py +0 -0
  93. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/prescriptive_analytics/__init__.py +0 -0
  94. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/probability/ProbabilityOfAtLeastOne.py +0 -0
  95. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/probability/__init__.py +0 -0
  96. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateMetalogDistributionFromPercentiles.py +0 -0
  97. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateSIPDataframe.py +0 -0
  98. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/CreateSLURPDistribution.py +0 -0
  99. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountOfSuccesses.py +0 -0
  100. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountOutcome.py +0 -0
  101. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateCountUntilFirstSuccess.py +0 -0
  102. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateNormallyDistributedOutcome.py +0 -0
  103. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTDistributedOutcome.py +0 -0
  104. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTimeBetweenEvents.py +0 -0
  105. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/SimulateTimeUntilNEvents.py +0 -0
  106. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/simulations/__init__.py +0 -0
  107. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/CalculateConfidenceIntervalOfMean.py +0 -0
  108. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/CalculateConfidenceIntervalOfProportion.py +0 -0
  109. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/statistics/__init__.py +0 -0
  110. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/Plot100PercentStackedBarChart.py +0 -0
  111. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBarChart.py +0 -0
  112. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBoxWhiskerByGroup.py +0 -0
  113. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotBulletChart.py +0 -0
  114. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotCard.py +0 -0
  115. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotClusteredBarChart.py +0 -0
  116. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotContingencyHeatmap.py +0 -0
  117. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotCorrelationMatrix.py +0 -0
  118. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotDensityByGroup.py +0 -0
  119. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotDotPlot.py +0 -0
  120. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotHeatmap.py +0 -0
  121. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotOverlappingAreaChart.py +0 -0
  122. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotRiskTolerance.py +0 -0
  123. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotScatterplot.py +0 -0
  124. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotSingleVariableCountPlot.py +0 -0
  125. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotSingleVariableHistogram.py +0 -0
  126. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/PlotTimeSeries.py +0 -0
  127. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/RenderTableOne.py +0 -0
  128. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox/visualizations/__init__.py +0 -0
  129. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/dependency_links.txt +0 -0
  130. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/requires.txt +0 -0
  131. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/analysistoolbox.egg-info/top_level.txt +0 -0
  132. {analysistoolbox-2.2.0 → analysistoolbox-2.2.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: analysistoolbox
3
- Version: 2.2.0
3
+ Version: 2.2.1
4
4
  Summary: A collection tools in Python for data collection and processing, statistics, analytics, and intelligence analysis.
5
5
  Home-page: https://github.com/KyleProtho/AnalysisToolBox/tree/master/analysistoolbox
6
6
  Author: Kyle Protho
@@ -87,7 +87,22 @@ def CreateMetalogDistribution(dataframe,
87
87
  # Extract values from the dataframe
88
88
  arr_variable = dataframe[variable]
89
89
  arr_variable = list(arr_variable)
90
+
91
+ # If there is a lower_bound, print a warning and filter the data
92
+ if lower_bound is not None:
93
+ if min(arr_variable) < lower_bound:
94
+ print(f"Warning: There are observations that are lower than the lower bound of {lower_bound}. Filtering data...")
95
+ arr_variable = [x for x in arr_variable if x >= lower_bound]
96
+
97
+ # If there is an upper_bound, print a warning and filter the data
98
+ if upper_bound is not None:
99
+ if max(arr_variable) > upper_bound:
100
+ print(f"Warning: There are observations that are higher than the upper bound of {upper_bound}. Filtering data...")
101
+ arr_variable = [x for x in arr_variable if x <= upper_bound]
90
102
 
103
+ # Turn off warnings
104
+ import warnings
105
+ warnings.filterwarnings("ignore")
91
106
  # Create a metalog distribution
92
107
  if lower_bound is None and upper_bound is None:
93
108
  metalog_dist = pm.metalog(
@@ -294,3 +309,34 @@ def CreateMetalogDistribution(dataframe,
294
309
  else:
295
310
  return arr_metalog
296
311
 
312
+
313
+ # Test the function using upper and lower bounds
314
+ from sklearn.datasets import load_iris
315
+ iris_df = pd.DataFrame(load_iris().data, columns=load_iris().feature_names)
316
+ CreateMetalogDistribution(
317
+ dataframe=iris_df,
318
+ variable='sepal length (cm)',
319
+ lower_bound=2,
320
+ upper_bound=7,
321
+ learning_rate=0.01,
322
+ term_maximum=9,
323
+ term_minimum=2,
324
+ term_for_random_sample=None,
325
+ number_of_samples=10000,
326
+ show_summary=True,
327
+ return_format='dataframe',
328
+ plot_metalog_distribution=True,
329
+ fill_color="#999999",
330
+ fill_transparency=0.6,
331
+ figure_size=(8, 6),
332
+ show_mean=True,
333
+ show_median=True,
334
+ title_for_plot="Metalog Distribution",
335
+ subtitle_for_plot="Showing the metalog distribution of the variable",
336
+ caption_for_plot=None,
337
+ data_source_for_plot=None,
338
+ show_y_axis=False,
339
+ title_y_indent=1.1,
340
+ subtitle_y_indent=1.05,
341
+ caption_y_indent=-0.15
342
+ )
@@ -0,0 +1,3 @@
1
+ from .metalog import metalog
2
+ from .class_method import rmetalog, plot, qmetalog, pmetalog, dmetalog, summary, update
3
+ name = "pymetalog"
@@ -0,0 +1,341 @@
1
+ import pandas as pd
2
+ import numpy as np
3
+ import scipy as sp
4
+
5
+ from scipy.optimize import linprog, minimize, NonlinearConstraint
6
+ from .pdf_quantile_functions import pdf_quantile_builder
7
+ from .support import diffMatMetalog, pdfMetalog, quantileMetalog, newtons_method_metalog
8
+
9
+ import time
10
+ import warnings
11
+
12
+ def a_vector_OLS_and_LP(m_dict,
13
+ bounds,
14
+ boundedness,
15
+ term_limit,
16
+ term_lower_bound,
17
+ fit_method,
18
+ alpha,
19
+ diff_error = .001,
20
+ diff_step = 0.001):
21
+
22
+ """ Main workhorse function of pymetalog package.
23
+ Called during metalog.__init__ method call.
24
+
25
+ Args:
26
+ m_dict (:obj:`dict` with keys ['params', 'dataValues', 'Y']): Initialized output_dict variable from metalog class.
27
+ - m_dict['params']: (:obj:`dict` with keys ['bounds', 'boundedness', 'term_limit', 'term_lower_bound', 'step_len', 'fit_method']):
28
+ * 'bounds': metalog.bounds
29
+ * 'boundedness': metalog.boundedness
30
+ * 'term_limit': metalog.term_limit
31
+ * 'term_lower_bound': metalog.term_lower_bound
32
+ * 'step_len': metalog.step_len
33
+ * 'fit_method': metalog.fit_method
34
+
35
+ - m_dict['dataValues']: (:obj:`pandas.DataFrame` with columns ['x','probs','z'] of type numeric):
36
+ * 'x': metalog.x
37
+ * 'probs': metalog.probs
38
+ * 'z': column calculated in metalog.append_zvector method
39
+ - depends on metalog.boundedness attribute
40
+ - metalog.boundedness = 'u':
41
+ * 'z' = metalog.x
42
+ - metalog.boundedness = 'sl':
43
+ * 'z' = log( (metalog.x-lower_bound) )
44
+ - metalog.boundedness = 'su':
45
+ * 'z' = = log( (upper_bound-metalog.x) )
46
+ - metalog.boundedness = 'b':
47
+ * 'z' = log( (metalog.x-lower_bound) / (upper_bound-metalog.x) )
48
+
49
+ - m_dict['Y']: (:obj:`pandas.DataFrame` with columns ['y1','y2','y3','y4', ... ,'yn'] of type numeric):
50
+ * 'y1': numpy.array of ones with length equal to len(x)
51
+ * 'y2': numpy.array of numeric values equal to the term attached to s in the logistic quantile function np.log(m_dict['dataValues']['probs'] / (1 - m_dict['dataValues']['probs']))
52
+ * 'y3': numpy.array of numeric values (m_dict['dataValues']['probs'] - 0.5) * m_dict['Y']['y2']
53
+ * 'y4': numpy.array of numeric values m_dict['Y']['y4'] = m_dict['dataValues']['probs'] - 0.5
54
+ * 'yn': numpy.array of numeric values:
55
+ - if n in 'yn' is odd,
56
+ m_dict['Y']['yn'] = m_dict['Y']['y4']**(int(i//2))
57
+ - if n in 'yn' is even,
58
+ zn = 'y' + str(n-1)
59
+ m_dict['Y'][yn] = m_dict['Y']['y2'] * m_dict['Y'][zn]
60
+
61
+ bounds (:obj:`list`): Upper and lower limits to filter the data with before calculating metalog quantiles/pdfs.
62
+ - should be set in conjunction with the `boundedness` parameter
63
+
64
+ boundedness (:obj:`str`): String that is used to specify the type of metalog to fit.
65
+ - must be in set ('u','sl','su','b')
66
+ - Default: 'u'
67
+ * Fits an unbounded metalog
68
+ - 'sl' fits a strictly lower bounded metalog
69
+ * len(bounds) must == 1
70
+ - 'su' fits a strictly upper bounded metalog
71
+ * len(bounds) must == 1
72
+ - 'b' fits a upper/lower bounded metalog
73
+ * len(bounds) must == 2
74
+ * bounds[1] must be > bounds[0]
75
+
76
+ term_limit (:obj:`int`): The upper limit of the range of metalog terms to use to fit the data.
77
+ - strictly > term_lower_bound
78
+ - in range [3,30]
79
+
80
+ term_lower_bound (:obj:`int`): The lower limit of the range of metalog terms to use to fit the data.
81
+ - strictly < term_limit
82
+ - in range [2,29]
83
+
84
+ fit_method (:obj:`str`): Fit method to use to fit metalog distribution.
85
+ - must be in set ('any','OLS','LP','MLE')
86
+ - Default: 'any'
87
+ * first tries 'OLS' method than 'LP'
88
+ - 'OLS' only tries to fit by solving directly for a coefficients using ordinary least squares method
89
+ - 'LP' only tries to estimate fit using simplex linear program optimization routine
90
+ - 'MLE' first tries 'OLS' method than falls back to a maximum likelihood estimation routine
91
+
92
+ alpha (:obj:`float`, optional): Regularization term to add to OLS fit
93
+ - strictly >= 0.
94
+ - should be set in conjunction with `penalty` parameter
95
+ - Default: 0. (no regularization, OLS)
96
+
97
+ diff_error (:obj:`float`, optional): Value used to in scipy.optimize.linprog method call
98
+ to init the array of values representing the
99
+ upper-bound of each inequality constraint (row) in A_ub.
100
+ - #TODO: Insert maths
101
+
102
+ diff_step (:obj:`float`, optional): Value passed to `step_len` parameter in support.py diffMatMetalog method call
103
+ defines the bin width for the Reimann sum of the differences differentiation method
104
+ - diffMatMetalog differentiates the metalog pdf
105
+ * Differentiation reference: https://math.stackexchange.com/a/313135
106
+ Returns:
107
+ m_dict: (:obj:`dict` with keys ['params', 'dataValues', 'Y', 'A', 'M', 'Validation'])
108
+ - m_dict['A']: (:obj:`pandas.DataFrame` with columns ['a2','a3', ... ,'an'] of type numeric):
109
+ * a2, a3, ... , an are our a coefficients returned by the method specified in `fit_method`
110
+
111
+ - m_dict['M']: (:obj:`pandas.DataFrame` with columns 0:'pdf_1',1:'cdf_1',2:'pdf_2',3:'cdf_2',
112
+ ...,((2*(term_limit-term_lower_bound))+1)-1:'pdf_n',
113
+ ((2*(term_limit-term_lower_bound))+1):'cdf_n'
114
+ where n is the total number of metalog fits determined by (term_limit-term_lower_bound)+1
115
+ )
116
+ * pdf_1, pdf_2, ... , pdf_n are the metalog pdfs returned by pdf_quantile_builder.pdfMetalog method
117
+ * cdf_1, cdf_2, ... , cdf_n are the metalog quantiles returned by pdf_quantile_builder.quantileMetalog method
118
+
119
+ - m_dict['y']: (:obj: `numpy.ndarray` of type float):
120
+ * Array of bin widths for both the pdf_n and cdf_n
121
+
122
+ - m_dict['Validation']: (:obj:`pandas.DataFrame` with columns ['term', 'valid', 'method'] of type str):
123
+ * 'term': each metalog estimation given a number of terms
124
+ * 'valid': boolean flag indicating if the metalog estimation was valid or not
125
+ * 'method': a string indicating which method was used for the metalog estimation
126
+
127
+ """
128
+
129
+ A = pd.DataFrame()
130
+ c_a_names = []
131
+ c_m_names = []
132
+ Mh = pd.DataFrame()
133
+ Validation = pd.DataFrame()
134
+ df_MH_temp_list = list()
135
+ df_A_temp_list = list()
136
+ df_Validation_temp_list = list()
137
+
138
+ # TODO: Large for-loop can probably be factored into smaller functions
139
+ for i in range(term_lower_bound,term_limit+1):
140
+ Y = m_dict['Y'].iloc[:,0:i]
141
+ eye = np.eye(Y.shape[1])
142
+ z = m_dict['dataValues']['z']
143
+ y = m_dict['dataValues']['probs']
144
+ step_len = m_dict['params']['step_len']
145
+ methodFit = 'OLS'
146
+ a_name = 'a'+str(i)
147
+ m_name = 'm'+str(i)
148
+ M_name = 'M'+str(i)
149
+ c_m_names = np.append(c_m_names, [m_name, M_name])
150
+ c_a_names = np.append(c_a_names, a_name)
151
+
152
+ if fit_method == 'any' or fit_method == 'MLE':
153
+ try:
154
+ temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
155
+ except:
156
+ # use LP solver if OLS breaks
157
+ temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
158
+ methodFit = 'Linear Program'
159
+ if fit_method == 'OLS':
160
+ try:
161
+ temp = np.dot(np.dot(np.linalg.inv(np.dot(Y.T, Y) + alpha*eye), Y.T), z)
162
+ except:
163
+ raise RuntimeError("OLS was unable to solve infeasible or poorly formulated problem")
164
+ if fit_method == "LP":
165
+ temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
166
+ methodFit = 'Linear Program'
167
+
168
+ if fit_method == 'MLE':
169
+ temp = a_vector_MLE(temp, y, i, m_dict, bounds, boundedness)
170
+
171
+ temp = np.append(temp, np.zeros(term_limit-i))
172
+
173
+ # build a y vector for smaller data sets
174
+ if len(z) < 100:
175
+ y2 = np.linspace(step_len, 1 - step_len, int((1 - step_len) / step_len))
176
+ tailstep = step_len / 10
177
+ y1 = np.linspace(tailstep, (min(y2) - tailstep), int((min(y2) - tailstep) / tailstep))
178
+ y3 = np.linspace((max(y2) + tailstep), (max(y2) + tailstep * 9), int((tailstep * 9) / tailstep))
179
+ y = np.hstack((y1, y2, y3))
180
+
181
+ # Get the dict and quantile values back for validation
182
+ temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)
183
+
184
+ # If it not a valid pdf run and the OLS version was used the LP version
185
+ if (temp_dict['valid'] == 'no') and (fit_method != 'OLS'):
186
+ temp = a_vector_LP(m_dict, term_limit=i, term_lower_bound=i, diff_error=diff_error, diff_step=diff_step)
187
+ temp = np.append(temp, np.zeros(term_limit-i))
188
+ methodFit = 'Linear Program'
189
+
190
+ # Get the dict and quantile values back for validation
191
+ temp_dict = pdf_quantile_builder(temp, y=y, term_limit=i, bounds=bounds, boundedness=boundedness)
192
+
193
+ df_MH_temp_list.append(pd.DataFrame(temp_dict['m']))
194
+ df_MH_temp_list.append(pd.DataFrame(temp_dict['M']))
195
+ df_A_temp_list.append(pd.DataFrame(temp))
196
+
197
+ tempValidation = pd.DataFrame(data={'term': [i], 'valid': [temp_dict['valid']], 'method': [methodFit]})
198
+ df_Validation_temp_list.append(tempValidation)
199
+
200
+ Validation = pd.concat(df_Validation_temp_list, axis=0)
201
+ Mh = pd.concat(df_MH_temp_list, axis=1)
202
+ A = pd.concat(df_A_temp_list, axis=1)
203
+
204
+ A.columns = c_a_names
205
+ Mh.columns = c_m_names
206
+
207
+ m_dict['A'] = A
208
+ m_dict['M'] = Mh
209
+ m_dict['M']['y'] = temp_dict['y']
210
+ m_dict['Validation'] = Validation
211
+
212
+ A = np.column_stack((np.repeat(1.,len(A)), A))
213
+ Est = np.dot(m_dict['Y'], A)
214
+ ncols = A.shape[1]
215
+ try:
216
+ Z = np.column_stack((np.array(m_dict['dataValues']['z']),np.repeat(m_dict['dataValues']['z'].values,ncols-1).reshape(len(m_dict['dataValues']['z']),ncols-1)))
217
+ except:
218
+ Z = np.column_stack((np.array(m_dict['dataValues']['z']),np.repeat(m_dict['dataValues']['z'], ncols-1).reshape(len(m_dict['dataValues']['z']),ncols-1)))
219
+
220
+ m_dict['square_residual_error'] = ((Z-Est)**2).sum(axis=1)
221
+
222
+ return m_dict
223
+
224
+ def a_vector_LP(m_dict, term_limit, term_lower_bound, diff_error = .001, diff_step = 0.001):
225
+ """TODO: write docstring
226
+
227
+ """
228
+ cnames = np.array([])
229
+
230
+ for i in range(term_lower_bound, term_limit + 1):
231
+ Y = m_dict['Y'].iloc[:, 0:i]
232
+ z = m_dict['dataValues']['z']
233
+
234
+ # Bulding the objective function using abs value LP formulation
235
+ Y_neg = -Y
236
+
237
+ new_Y = pd.DataFrame({'y1': Y.iloc[:, 0], 'y1_neg': Y_neg.iloc[:, 0]})
238
+
239
+ for c in range(1,len(Y.iloc[0,:])):
240
+ new_Y['y'+str(c+1)] = Y.iloc[:,c]
241
+ new_Y['y' + str(c+1)+'_neg'] = Y_neg.iloc[:, c]
242
+
243
+ a = np.array([''.join(['a', str(i)])])
244
+ cnames = np.append(cnames, a, axis=0)
245
+
246
+ # Building the constraint matrix
247
+ error_mat = np.array([])
248
+
249
+ for j in range(1,len(Y.iloc[:,0])+1):
250
+ front_zeros = np.zeros(2 * (j - 1))
251
+ ones = [1, -1]
252
+ trail_zeroes = np.zeros(2 * (len(Y.iloc[:, 1]) - j))
253
+ if j == 1:
254
+ error_vars = np.append(ones, trail_zeroes)
255
+
256
+ elif j != 1:
257
+ error_vars = np.append(front_zeros, ones)
258
+ error_vars = np.append(error_vars, trail_zeroes)
259
+
260
+ if error_mat.size == 0:
261
+ error_mat = np.append(error_mat, error_vars, axis=0)
262
+ else:
263
+ error_mat = np.vstack((error_mat, error_vars))
264
+
265
+ new = pd.concat((pd.DataFrame(data=error_mat), new_Y), axis=1)
266
+ diff_mat = diffMatMetalog(i, diff_step)
267
+ diff_zeros = []
268
+
269
+ for t in range(0,len(diff_mat.iloc[:, 0])):
270
+ zeros_temp = np.zeros(2 * len(Y.iloc[:, 0]))
271
+
272
+ if np.size(diff_zeros) == 0:
273
+ diff_zeros = zeros_temp
274
+ else:
275
+ diff_zeros = np.vstack((zeros_temp, diff_zeros))
276
+
277
+ diff_mat = np.concatenate((diff_zeros, diff_mat), axis=1)
278
+
279
+ # Combine the total constraint matrix
280
+ lp_mat = np.concatenate((new, diff_mat), axis=0)
281
+
282
+ # Objective function coeficients
283
+ c = np.append(np.ones(2 * len(Y.iloc[:, 1])), np.zeros(2*i))
284
+
285
+ # Constraint matrices
286
+ A_eq = lp_mat[:len(Y.iloc[:, 1]),:]
287
+ A_ub = -1*lp_mat[len(Y.iloc[:, 1]):,:]
288
+ b_eq = z
289
+ b_ub = -1*np.repeat(diff_error, len(diff_mat[:,0]))
290
+
291
+ # Solving the linear program w/ scipy (for now)
292
+ lp_sol = linprog(c, A_ub=A_ub, b_ub=b_ub, A_eq=A_eq, b_eq=b_eq, method='simplex', options={"maxiter":5000, "tol":1.0e-5,"disp": False})
293
+
294
+ # Consolidating solution back into the a vector
295
+ tempLP = lp_sol.x[(2 * len(Y.iloc[:, 1])):(len(lp_sol.x)+1)]
296
+ temp = []
297
+
298
+ for r in range(0,((len(tempLP) // 2))):
299
+ temp.append(tempLP[(r * 2)] - tempLP[(2 * r)+1])
300
+
301
+ return temp
302
+
303
+
304
+ def a_vector_MLE(a, y, term, m_dict, bounds, boundedness):
305
+ """TODO: write docstring
306
+
307
+ """
308
+ ym = [newtons_method_metalog(a, xi, term, bounds, boundedness) for xi in m_dict['dataValues']['x']]
309
+
310
+ def MLE_quantile_constraints(x):
311
+ M = [quantileMetalog(x[:term], yi, term, bounds=bounds, boundedness=boundedness) for yi in x[term:]]
312
+ return m_dict['dataValues']['x'] - M
313
+
314
+ def MLE_objective_function(x, y, term, m_dict):
315
+ return -np.sum([np.log10(pdfMetalog(x[:term], yi, term, bounds, boundedness)) for yi in np.absolute(x[term:])])
316
+
317
+ m_dict[str('MLE' + str(term))] = {}
318
+
319
+ x0 = np.hstack((a[:term],ym))
320
+ m_dict[str('MLE' + str(term))]['oldobj'] = -MLE_objective_function(x0, y, term, m_dict)
321
+ bnd = ((None, None),)*len(a)+((0, 1),)*(len(x0)-len(a))
322
+ con = NonlinearConstraint(MLE_quantile_constraints, 0, 0)
323
+
324
+ mle = minimize(MLE_objective_function, x0, args=(y, term, m_dict), bounds=bnd, constraints=con)
325
+
326
+ m_dict[str('MLE' + str(term))]['newobj'] = -MLE_objective_function(mle.x, y, term, m_dict)
327
+ m_dict[str('MLE'+str(term))]['A'] = mle.x[:term]
328
+ m_dict[str('MLE'+str(term))]['Y'] = mle.x[term:]
329
+
330
+ m_dict[str('MLE' + str(term))]['oldA'] = a
331
+ m_dict[str('MLE' + str(term))]['oldY'] = y
332
+
333
+ out_temp = np.zeros_like(a)
334
+ for i in range(term):
335
+ out_temp[i] = mle.x[i]
336
+
337
+ return out_temp
338
+
339
+
340
+
341
+