SweetSpot 1.0.1__py2.py3-none-any.whl

SweetSpot/AdaptiveProcedures.py

@@ -0,0 +1,791 @@
+# $BEGIN_SWEETSPOT_LICENSE$
+# 
+# This file is part of the SweetSpot project, a Python package that
+# implements adaptive psychophysical procedures with minimal dependencies.
+# 
+# Author: Jeremy Hill (2018-)
+# Development was supported by the NIH, NYS SCIRB, Veterans Affairs RRD,
+# and the Stratton VA Medical Center.
+# 
+# No Copyright
+# ============
+# The author has dedicated this work to the public domain under the terms of
+# Creative Commons' CC0 1.0 Universal legal code, waiving all of his rights to
+# the work worldwide under copyright law, including all related and neighboring
+# rights, to the extent allowed by law.
+# 
+# You can copy, modify, distribute and perform the work, even for commercial
+# purposes, all without asking permission. See Other Information below.
+# 
+# Other Information
+# =================
+# In no way are the patent or trademark rights of any person affected by CC0,
+# nor are the rights that other persons may have in the work or in how the work
+# is used, such as publicity or privacy rights.
+# 
+# The author makes no warranties about the work, and disclaims liability for
+# all uses of the work, to the fullest extent permitted by applicable law. When
+# using or citing the work, you are requested to preserve the author attribution
+# and this copyright waiver, but you should not imply endorsement by the author.
+# 
+# $END_SWEETSPOT_LICENSE$
+"""
+This submodule defines the abstract `AdaptiveProcedure` base class and two subclasses:
+`WUD` (Kaernbach 1991) and `PsiMarginal` (Prins 2013). It also provides the global
+function `Simulate()` for performance testing.
+"""
+
+from __future__ import absolute_import
+
+__all__ = [
+	'AdaptiveProcedure',  # generic superclass
+	'WUD', 'PsiMarginal', # two example subclasses
+	'Simulate',
+]
+
+import math
+import itertools
+
+import numpy   # NB: matplotlib is also imported in the particular methods that need it
+
+from .PsychometricFunctions import PsychophysicalData, AbstractClassError # used in the AdaptiveProcedure class
+from .PsychometricFunctions import PsychometricFunction, Logistic, Chain  # used in the Simulate() function
+from .MPLExtras import ResolveFigure, ResolveAxes, UpdateAxesTitle, MakeSubplots, PdfMaker, KeyboardInteractor, RaiseFigure
+
+def ModeEstimate( pr, values, tolerance=1e-10 ):
+	maxPr = max( pr.flat )
+	return float( numpy.median( [ v for v, p in zip( values.flat, pr.flat ) if abs( p - maxPr ) <= tolerance ] ) )
+	
+class AdaptiveProcedure( object ):
+	"""
+	Abstract superclass for a number of different adaptive procedures
+	"""
+	isStaircase = property( lambda self: True )
+	def __init__( self, start, min=None, max=None ):
+		self.__start = None
+		self.__pendingLevel = None
+		self.__dataXRN = []
+		self.__reversals = []
+		self.__relationships = {}
+		self.min = min
+		self.max = max
+		self.Reset( start )
+	
+	def Reset( self, start=None ):
+		if start is not None: self.__start = start
+		del self.__reversals[ : ]
+		del self.__dataXRN[ : ]
+		self.__pendingLevel = self.__start
+		return self.__pendingLevel
+	
+	def Observe( self, r, n=1, x=None ):
+		"""
+		When you have performed a new trial, call:
+		
+		- `.Observe(1)` to record a correct (or positive) response, or
+		- `.Observe(0)` to record an incorrect (or negative) response.
+		
+		The trial will be assumed to have occurred with stimulus intensity set to the
+		current `.PendingValue()`, unless you explicitly supply its value as `x`.
+		
+		By default, it will be assumed that `n=1` trials were performed at that intensity,
+		but you can record a larger batch of trials by specifying a different `n`, in
+		which case the first argument `r` should be equal to the number of correct responses.
+		(In the internal logic for monitoring "reversals", an observation will be considered
+		"successful" if r >= n/2.)
+		
+		After recording the response, the adaptive procedure will automatically check
+		whether it is `.Finished()` - if so, this method will return `None`; if not, the
+		procedure will call its `.ComputeNextValue()` method and store the result as the
+		next pending value, which this method will then return.
+		"""
+		if self.Finished(): return None
+		if n <= 0: return self.__pendingLevel
+		outcomeIndex = len( self.__dataXRN )
+		if outcomeIndex > 0: # this is the logic for computing whether a trial is a reversal *before* recording nHits and nTrials (so it does not call .Successful() or .IsReversal(), which are designed to be called after)
+			xPrev, rPrev, nPrev = self.__dataXRN[ -1 ]
+			successfulLastTime = ( rPrev >= 0.5 * nPrev )
+			successfulThisTime = ( r     >= 0.5 * n     )
+			if successfulThisTime != successfulLastTime:
+				self.__reversals.append( outcomeIndex )
+		if x is None: x = self.__pendingLevel
+		self.__dataXRN.append( ( float( x ), float( r ), float( n ) )  )
+		if self.Finished():
+			next = None
+		else:
+			next = float( self.ComputeNextValue() )
+			if self.min is not None: next = max( self.min, next )
+			if self.max is not None: next = min( self.max, next )
+		self.__pendingLevel = next
+		return next
+
+	def Relate( self, name, base, hitFactor, floor=None, ceiling=None ):
+		self.__relationships[ name ] = dict( base=float( base ), hitFactor=float( hitFactor ), floor=floor, ceiling=ceiling )
+	
+	def PhysicalToDetectability( self, physicalValue, relationship ):
+		r = self.__relationships[ relationship ]
+		physicalValue = numpy.array( physicalValue, dtype=float )
+		valid = numpy.array( [ True ] * physicalValue.size )
+		detectabilityValue = physicalValue * numpy.nan
+		if r[ 'floor' ] is not None: valid[ physicalValue.ravel() < r[ 'floor' ] ] = False
+		if r[ 'ceiling' ] is not None: valid[ physicalValue.ravel() > r[ 'ceiling' ] ] = False
+		
+		if r[ 'hitFactor' ] == 1.0: valid.flat = False
+		else: detectabilityValue.flat[ valid ] = numpy.log( r[ 'base' ] / physicalValue[ valid ].ravel() ) / numpy.log( r[ 'hitFactor' ] )
+		
+		try: detectabilityValue = float( detectabilityValue )
+		except: pass
+		return detectabilityValue
+		
+	def DetectabilityToPhysical( self, detectabilityValue, relationship=None ):
+		if relationship is None: return dict( [ ( k, self.DetectabilityToPhysical( detectabilityValue, k ) ) for k in self.__relationships ] )
+		r = self.__relationships[ relationship ]
+		detectabilityValue = numpy.array( detectabilityValue, dtype=float )
+		
+		physicalValue = r[ 'base' ] * r[ 'hitFactor' ] ** -detectabilityValue
+		
+		if r[ 'floor' ] is not None: physicalValue = physicalValue.clip( min=r[ 'floor' ] ) # NB: this is done in two lines because .clip() doesn't seem to behave
+		if r[ 'ceiling' ] is not None: physicalValue = physicalValue.clip( max=r[ 'ceiling' ] ) # correctly if performed all in one call with min=nonNone, max=None
+		try: physicalValue = float( physicalValue )
+		except: pass
+		return physicalValue
+		
+	def Observation( self, trialIndex=-1 ):
+		try: x, r, n = self.__dataXRN[ trialIndex ]
+		except IndexError: return None, None, None
+		return x, r, n
+	
+	def IsReversal( self, trialIndex=-1 ): # NB: this is designed as a utility to be called *only* from a subclass's ComputeNextValue - i.e. after the current outcome has been recorded
+		if trialIndex < 0: trialIndex += len( self.__dataXRN )
+		return trialIndex in self.__reversals
+	
+	def Successful( self, trialIndex=-1 ): # NB: this is designed as a utility to be called *only* from a subclass's ComputeNextValue - i.e. after the current outcome has been recorded
+		try: x, r, n = self.__dataXRN[ trialIndex ]
+		except IndexError: return None
+		return r >= n / 2.0
+	
+	def Data( self ): return PsychophysicalData( data=self.__dataXRN, format='xrn' )
+	def NumberOfPoints( self ): return len( self.__dataXRN )
+	def Reversals( self ): return tuple( self.__reversals )
+	def PendingValue( self ): return self.__pendingLevel
+	def OverridePendingValue( self, value ): self.__pendingLevel = value
+	def PendingStep( self ):
+		if len( self.__dataXRN ) < 1 or self.__pendingLevel is None: return None
+		return self.__pendingLevel - self.__dataXRN[ -1 ][ 0 ]
+	
+	def Label( self ):
+		return self.__class__.__name__
+
+	@property
+	def data( self ):
+		return self.Data()
+			
+	# overshadow these:
+	def Finished( self ): raise AbstractClassError( self )
+	def ComputeNextValue( self ): raise AbstractClassError( self )
+	def Estimate( self ): raise AbstractClassError( self )
+	
+	#####################################################################################
+	
+	def Test( self, truePF, start='random', n=1, callback=None, parallel=() ):
+		"""
+		`truePF` can either be a `PsychometricFunction` instance, or a `PsychophysicalData`
+		instance---the latter case is just "playback".
+		
+		`start` can be a numeric stimulus intensity value, or it can be `None` (in which
+		case the first stimulus value is the current `.PendingValue()`), or it can be
+		'random'.  Note that 'random' consults the *true* psychometric function to come
+		up with a ballpark of where to start---this tacitly assumes some prior knowledge
+		about the psychometric function. This is basically a quick hack for non-Bayesian
+		methods. If you're using a Bayesian method, prior knowledge would be better
+		expressed by being encoding properly in the priors of the `PsychometricFunction`
+		instance with which the `AdaptiveProcedure` was initialized, and then letting the
+		procedure do its thing naturally with `start=None`.
+		
+		`n` is the number of binary trials per batch of observations. It can be a scalar,
+		or it can be a sequence.  If it is a sequence, the number of batches is capped
+		at the number of elements of `n`. In either case, the procedure will also stop if
+		its `.Finished()` method returns true.
+		
+		`callback`, if provided, is called before each observation (or batch of
+		observations). When called, its sole argument will be the `AdaptiveProcedure`
+		instance `self`. If it returns a true value, the test will abort (this can be
+		used, for example, to allow the user to abort manually via an interactive prompt).
+		
+		`parallel`, if provided, should be a sequence of `AdaptiveProcedure` instances.
+		Each such instance also has `.Observe()` called for each batch of observations.
+		This can be useful, for example, to allow a `PsiMarginal` instance to track the
+		expected entropy of the parameters, even when the stimulus intensity decisions
+		are being driven by a different algorithm `self`.		
+		"""
+		
+		playbackData = None
+		parallel = set( parallel )
+		parallel.add( self )
+		if isinstance( truePF, PsychophysicalData ):
+			playbackData, truePF = truePF, None
+			n = playbackData.n.flat
+			start = playbackData.x.flat[ 0 ]
+		else:
+			try: len( n )
+			except: n = itertools.repeat( n )
+			if start == 'random':
+				lower, = truePF.Inverse( f=0.5 ).flat
+				upper, = truePF.Inverse( f=0.95 ).flat
+				upper = lower + ( upper - lower ) * 3
+				mean  = ( lower + upper ) / 2.0
+				std   = ( upper - lower ) / 6.0
+				start = numpy.random.normal() * std + mean
+		for ap in parallel: ap.Reset( start=start )
+		for i, ni in enumerate( n ):
+			if callback and callback( self ): break
+			if self.Finished(): break
+			if playbackData:
+				xi = playbackData.x.flat[ i ]
+				ri = playbackData.r.flat[ i ]
+			else:
+				xi = self.PendingValue()
+				ri = truePF.SimulatedResponse( x=xi, n=ni )
+			for ap in parallel: ap.Observe( x=xi, r=ri, n=ni )
+		return self
+	
+	def PlotStaircase( self, figure=None, axes=None, hold=False, title=None, raiseFigure=False, **kwargs ):
+		xline, yline, xhits, yhits, xmisses, ymisses, xreversals, yreversals = [], [], [], [], [], [], [], []
+		for i, ( x, r, n ) in enumerate( self.__dataXRN ):
+			xline += [ i, i + 1 ]; yline += [ float( x ), float( x ) ]
+			if self.Successful( i ): xhits.append( i + 1 ); yhits.append( float( x ) )
+			else:                    xmisses.append( i + 1 ); ymisses.append( float( x ) )
+			if self.IsReversal( i ): xreversals.append( i + 1 ); yreversals.append( float( x ) )
+		axes = ResolveAxes( figure=figure, axes=axes, hold=hold )
+		args = dict( kwargs ); args[ 'marker' ] = 'None'
+		line, = axes.plot( xline, yline, **args )
+		color = line.get_color()
+		kwargs.setdefault( 'color', color )
+		kwargs.setdefault( 'marker', 's' )
+		kwargs.setdefault( 'markersize', 10 )
+		kwargs.update( dict( linestyle='none', mec=color, mfc=color ) )
+		hits, = axes.plot( xhits, yhits, **kwargs )
+		kwargs.update( dict( mfc='w' ) )
+		misses, = axes.plot( xmisses, ymisses, **kwargs )
+		kwargs.update( dict( mfc='none', markersize=kwargs[ 'markersize' ] * 2 ) )
+		reversals, = axes.plot( xreversals, yreversals, **kwargs )
+		if self.Finished():
+			est = self.Estimate()
+			if isinstance( est, float ):
+				kwargs.update( dict( marker='None', linestyle='--', linewidth=2 ) )
+				estimate = axes.plot( [0, i + 1], [ est, est ], **kwargs )
+		axes.set( xlabel='Trial Number', ylabel='Stimulus Level' )
+		if title: axes.set_title( title )
+		if raiseFigure: RaiseFigure( axes.figure )
+				
+class WUD( AdaptiveProcedure ):
+	"""
+	The Weighted Up-Down of Kaernbach (1991).
+
+	- Kaernbach C (1991): Simple adaptive testing with the weighted up-down method.
+	  Perception & Psychophysics 49(3):227-9. https://doi.org/10.3758/bf03214307
+	
+	WUD( start=None, hitEffect=-0.1 ).Test( PsychometricFunction( gamma=0.01, lambda_=0.01 ) ).PlotStaircase()
+	"""
+	isStaircase = property( lambda self: True )
+	def __init__( self, start, hitEffect, target=0.75, reversals=8, reversalsToUse=6, multiplicative=False ):
+		self.hitEffect = hitEffect
+		self.target = target
+		self.numberOfReversals = reversals
+		self.reversalsToUse = reversalsToUse
+		self.multiplicative = multiplicative
+		if self.multiplicative and self.hitEffect <= 0.0: raise ValueError( 'in multiplicative mode, hitEffect must be > 0' )
+		if self.hitEffect == 0.0 and not self.multiplicative: raise ValueError( 'hitEffect=0.0 has no effect in additive mode' )
+		if self.hitEffect == 1.0 and self.multiplicative:     raise ValueError( 'hitEffect=1.0 has no effect in multiplicative mode' )
+		AdaptiveProcedure.__init__( self, start=start )
+	
+	def Reset( self, start=None ):
+		self.adjustedHitEffect = self.hitEffect
+		AdaptiveProcedure.Reset( self, start=start )
+		
+	def ToAdditiveSpace(   self, value ): return math.log( value ) if self.multiplicative else value
+	def FromAdditiveSpace( self, value ): return math.exp( value ) if self.multiplicative else value
+
+	def ComputeNextValue( self ):
+		if self.IsReversal():
+			revNumber = len( self.Reversals() ) # (one-based) reversal counter
+			# TODO: could include a rule for narrowing self.adjustedHitEffect after a certain number of reversals based on revNumber
+		step = self.ToAdditiveSpace( self.adjustedHitEffect )
+		xPrev, _, _ = self.Observation( trialIndex=-1 )
+		if not self.Successful( trialIndex=-1 ): step *= -self.target / ( 1.0 - self.target )
+		return self.FromAdditiveSpace( self.ToAdditiveSpace( xPrev ) + step )
+	
+	def Finished( self ):
+		return len( self.Reversals() ) >= self.numberOfReversals
+		
+	def Estimate( self, mode='median' ):
+		if not self.Finished(): raise RuntimeError( "staircase is not finished" )
+		if mode == 'all': return [ ( mode, self.Estimate( mode ) ) for mode in [ 'median' ] ]
+		reversals = [ self.Observation( index )[ 0 ] for index in self.Reversals()[ -self.reversalsToUse: ] ]
+		if mode == 'median': return float( numpy.median( reversals ) )
+		raise ValueError( 'unrecognized mode %r' % mode )
+
+
+class PsiMarginal( AdaptiveProcedure ):
+	"""
+	A discrete grid approximation to the psi-marginal method of Prins (2013).
+	
+	- Prins N (2013): The psi-marginal adaptive method: How to give nuisance
+	  parameters the attention they deserve (no more, no less).
+	  Journal of Vision 13(7):3.  https://doi.org/10.1167/13.7.3
+	  
+	"""
+	isStaircase = property( lambda self: False )
+	def __init__( self, pf, xCandidates, start='auto', maxIterations=50, blockSize=1, mode='MAP', mercy=0.0, maxNegativeStep=None, pdfFile=None, parameterOfInterest='alpha' ):
+		"""
+		The first argument, `pf`, is a PsychometricFunction instance which must be
+		initialized with:
+		
+		- a shape, operating point and experimental design, e.g.
+		  `pf = PsychometricFunction( shape=Logistic( 0.5 ), gamma=0.5 )`
+		  
+		- a search grid, defined via `pf.Grid()`
+		
+		- prior information on parameter spread, defined via `pf.SetPrior()` (unless
+		  you explicitly and genuinely want flat priors)
+		
+		  
+		The second argument `xCandidates`, should be a discrete array of possible
+		stimulus intensity values to consider.
+		
+		The `mode` argument can be 'MLE', 'MAP' or 'AVG' and it does no more than
+		dictate the default value of the `mode` argument to `.Estimate()`.
+		
+		The `.ComputeNextValue()` method chooses the stimulus value that minimizes
+		expected posterior entropy. Two options, implemented in the `.Psychology()`
+		method, can be turned on to modulate this slightly, in an attempt to minimize
+		the psychological impact of order effects:
+		
+		- with `mercy > 0.0`, following an incorrect response, "difficult" stimulus
+		  values (i.e. those for whom `F(x)<mercy` according to current estimates,
+		  or harder than the one you just got wrong) are not considered, in an
+		  attempt to avoid subjects getting too discouraged.
+		
+		- if `maxNegativeStep` (default: `None`) is set to a numeric value, then the
+		  stimulus intensity will never be reduced (or, more accurately, will never be
+		  changed in the direction that `self.pf.flipX` dictates is "more difficult")
+		  by more than the specified amount in a single step.
+				
+		"""
+		self.pf = pf
+		self.xCandidates = xCandidates
+		self.parameterOfInterest = parameterOfInterest
+		self.maxIterations = maxIterations
+		self.maxNegativeStep = maxNegativeStep
+		self.blockSize = blockSize
+		self.mode = mode.upper()
+		self.pdfFile = None
+		self.pdfFileName = None
+		self.entropyHistory = []
+		self.mercy = mercy
+		if self.mode not in [ 'MAP', 'MLE', 'AVG' ]: raise ValueError( "mode should be 'MAP', 'MLE' or 'AVG'" )
+		if start is None: start = 'auto'
+		if pdfFile:
+			self.pdfFileName = pdfFile
+			self.pdfFile = PdfMaker.MultipagePdfHandle( pdfFile )
+		AdaptiveProcedure.__init__( self, start=start )
+
+	@property
+	def mercy( self ):
+		"""
+		A value in the range [0.0, 1.0) - non-zero values enable a heuristic that
+		attempts to prevent subjects getting overly discouraged by failure (see
+		class constructor doc).
+		"""
+		return self.__mercy
+	@mercy.setter
+	def mercy( self, value ):
+		if isinstance( value, bool ) and value: value = 0.5 # default value for mercy=True
+		if not ( value < 1.0 ): raise ValueError( 'mercy must be < 1.0' )
+		if value < 0.0: raise ValueError( 'mercy cannot be negative (you scare me)' )
+		self.__mercy = value
+		
+	def Label( self ):
+		t = self.__class__.__name__
+		poi = self.parameterOfInterest
+		if isinstance( poi, str ): poi = poi.replace( ',', ' ' ).split()
+		t += '(%s)' % ','.join( poi )
+		return t
+
+	def Observe( self, r, n=1, x=None ):
+		if x is None: x = self.PendingValue()
+		self.pf.Observe( x=x, r=r, n=n )
+		result = AdaptiveProcedure.Observe( self, r=r, n=n, x=x )
+		self.entropyHistory.append( self.entropy )
+		PdfMaker.DrawToFile( self.pdfFile, self.PlotPredictions )
+		if self.Finished(): self.ClosePdfFile()
+		return result
+	
+	def Reset( self, start=None ):
+		self.pf.Reset()
+		auto = ( isinstance( start, str ) and start == 'auto' )
+		if auto: start = None
+		AdaptiveProcedure.Reset( self, start=start ) # blank slate
+		recommendation = self.ComputeNextValue() # computes entropy etc as a side effect
+		if auto: AdaptiveProcedure.Reset( self, start=recommendation )
+		self.entropyHistory[ : ] = [ self.entropy ]
+		PdfMaker.DrawToFile( self.pdfFile, self.pf.VisualizeDistributions, self.PlotPredictions )
+		
+	def Finished( self ):
+		#uncertainty = self.entropy
+		# TODO: could set a criterion based on uncertainty
+		return self.NumberOfPoints() >= self.maxIterations
+		
+	def ComputeNextValue( self ):
+		self.expectedEntropy = self.pf.ExpectedEntropy( x=self.xCandidates, n=self.blockSize, params=self.parameterOfInterest )
+		self.adjustedExpectedEntropy = self.expectedEntropy.copy()
+		self.Psychology() # if this method finds self.adjustedExpectedEntropy, it will adjust it
+		return self.xCandidates[ numpy.nanargmin( self.adjustedExpectedEntropy ) ]
+	
+	def WhicheverIsHardest( self, *x ):
+		x = [ xi for xi in x if xi is not None ]
+		if len( x ) == 0: return None
+		if len( x ) == 1:
+			try: float( x[ 0 ] )
+			except: pass
+			else: return x[ 0 ]
+		return max( *x ) if self.pf.flipX else min( *x )
+	def WhicheverIsEasiest( self, *x ):
+		x = [ xi for xi in x if xi is not None ]
+		if len( x ) == 0: return None
+		if len( x ) == 1:
+			try: float( x[ 0 ] )
+			except: pass
+			else: return x[ 0 ]
+		return min( *x ) if self.pf.flipX else max( *x )
+	def IsHarder   ( self, x, thanWhat):  return ( x > thanWhat  ) if self.pf.flipX else ( x < thanWhat  )
+	def IsEasier   ( self, x, thanWhat):  return ( x < thanWhat  ) if self.pf.flipX else ( x > thanWhat  )
+	def MakeHarder ( self, x, byHowMuch): return ( x + byHowMuch ) if self.pf.flipX else ( x - byHowMuch )
+	def MakeEasier ( self, x, byHowMuch): return ( x - byHowMuch ) if self.pf.flipX else ( x + byHowMuch )
+	
+	def Psychology( self ):
+		"""
+		Overshadowable method implementing non-Bayesian heuristics that are theoretically
+		non-optimal but which we suspect help to patch non-stationarities in human behavior.
+		"""
+		lastX, lastR, lastN = self.Observation( -1 )
+		hardestAllowableX = []
+		debug = False
+		
+		# (A) `mercy > 0.0` means "do not present a 'difficult' stimulus immediately after a miss".
+		#     The motivation is to avoid subjects losing heart (the short-term learned helplessness of
+		#     "this is stupid, there's nothing on the screen!"):
+		if self.mercy > 0.0:
+			psychologicalF = self.mercy # let's use an absolute definition of the operating point for psychological
+			                            # purposes, rather than whatever the self.pf.shape happens to have configured.
+			                            # The value we choose here is a psychological hyperparameter, up for debate...
+			                            # 0.5 might be a sensible value to try.
+			cutoffPerformanceLevel = self.pf.OperatingPoint( psychologicalF, scaled=True )
+			if lastX is not None and float( lastR ) / lastN < cutoffPerformanceLevel: # if the last observation can be considered negative
+				
+				# Two criteria are applied, to exclude stimuli that are...
+				# (1) harder than the one you just missed:
+				hardestAllowableX.append( lastX )
+				if debug: print( '%+4.2f is a bound because that was the value of the missed stimulus' % hardestAllowableX[ -1 ] )
+				
+				# or (2) predicted "hard" according to the model's best (marginalized) guess;
+				
+				# Here is the old method - evaluate the marginalized prediction at all xCandidates and compare
+				# against the cutoff - which only works for entropy-/candidate-based methods. For other methods,
+				# (what we *really* want to do is invert the marginalized prediction, which we could in principle
+				# do by a bisection algorithm...)
+				#xCandidates = numpy.asarray( self.xCandidates )
+				#marginalizedPrediction = self.pf.MarginalizedPrediction( xCandidates )
+				#allowedX = xCandidates[ marginalizedPrediction >= cutoffPerformanceLevel ]
+				#hardestAllowableX.append( self.WhicheverIsHardest( *allowedX ) )
+				# NB: the even the MarginalizedPrediction(xCandidates) call above is somewhat inefficient in
+				#     entropy-based procedures since something very similar to this will have been computed
+				#     inside ExpectedEntropy() (but note that it is only executed here after negative trials).
+				#     It added about 170ms in one 2018/2019 measurement with resolutions
+				#     x=100, alpha=100, beta=10, gamma=6, lambda_=6
+				
+				# ...but a quicker general solution than inverting the marginalized prediction (not exactly
+				# equivalent, but hey we're in heuristicville anyway) is to get a single set of
+				# posterior-averaged parameter values and invert the single function they define:
+				averagePF = PsychometricFunction(
+					shape = self.pf.shape,
+					preprocessing = self.pf.preprocessing,
+					flipX = self.pf.flipX,
+					**self.pf.AveragedParameters( 'all', 'posterior', asDict=True )
+				)
+				hardestAllowableX.append( float( averagePF.Inverse( f=psychologicalF ).item() ) )
+				
+				if debug: print( '%+4.2f is a bound because anything beyond that is probably "hard"' % hardestAllowableX[ -1 ] )
+
+
+		# (B) `maxNegativeStep` can be used to prevent surprisingly/offputtingly large sudden increases
+		#     in difficulty:
+		if lastX is not None and self.maxNegativeStep:
+			deltaX = abs( self.maxNegativeStep )
+			hardestAllowableX.append( self.MakeHarder( lastX, deltaX ) )
+			if debug: print( '%+4.2f is a bound because anything beyond that would be too big a step' % hardestAllowableX[ -1 ] )
+		
+		hardestAllowableX = self.WhicheverIsEasiest( *hardestAllowableX )
+		if hardestAllowableX is not None and getattr( self, 'adjustedExpectedEntropy', None ) is not None:
+			if debug: print( 'final exclusion cutoff: %+4.2f' % hardestAllowableX )
+			exclude = self.IsHarder( numpy.asarray( self.xCandidates ), hardestAllowableX )
+			self.adjustedExpectedEntropy[ exclude ] = numpy.nan
+			# if this wipes out the minimum, an entropy-based procedure is free to choose the next local minimum, which is not necessarily hardestAllowableX
+				
+		return hardestAllowableX # non entropy-based methods can use this return value to clamp their stimulus recommendation
+		
+	def Estimate( self, mode=None, params=None ):
+		if params is None: params = self.parameterOfInterest
+		if isinstance( params, str ): params = params.replace( ',', ' ' ).split()
+		
+		if mode is None: mode = self.mode
+		if   mode.upper() == 'ALL': return [ ( mode, self.Estimate( mode ) ) for mode in 'MLE MAP AVG'.split() ]
+		elif mode.upper() == 'AVG':
+			avg = self.pf.AveragedParameters( params, 'posterior', asDict=True )
+			if len( params ) == 1: return list( avg.values() )[ 0 ]
+			else: return avg
+		elif mode.upper() == 'CI':
+			if len( params ) > 1: raise NotImplementedError( 'CI mode only implemented for one parameterOfInterest at a time' ) # need to find the maximum over the multi-dimensional distro and the respective parameter values corresponding to the coordinates of that point in the different dimensions
+			return self.pf.PosteriorConfidenceInterval( params, 0.95 )
+		elif mode.upper() == 'MAP': distro = self.pf.LogPosterior()
+		elif mode.upper() == 'MLE': distro = self.pf.LogScore()
+		else: raise ValueError( 'unknown mode %r' % mode )
+		distro = self.pf.Distribution( distro, params=params )
+		if len( params ) == 1:
+			paramIndex, possibleParameterValues = self.pf.FindParameter( params[ 0 ] )
+			return ModeEstimate( distro.ravel(), possibleParameterValues.ravel() )
+		else:
+			maxLocation = numpy.unravel_index( numpy.argmax( distro ), distro.shape )
+			best = {}
+			for param in params:
+				paramIndex, possibleParameterValues = self.pf.FindParameter( param )
+				best[ param ] = float( possibleParameterValues.flat[ maxLocation[ paramIndex + 1 ] ] )
+			return best
+	
+	def PlotInference( self, parameterName, figure=None, axes=None ):
+		possibleParameterValues = self.pf.FindParameter( parameterName )[ 1 ].ravel()
+		distros = dict(
+			prior = self.pf.Distribution( self.pf.LogPrior(), params=parameterName ).ravel(),
+			likelihood = self.pf.Distribution( self.pf.LogScore(), params=parameterName ).ravel(),
+			posterior = self.pf.Distribution( self.pf.LogPosterior(), params=parameterName ).ravel(),
+		)
+		axes = ResolveAxes( figure=figure, axes=axes, hold=False )
+		for k, v in distros.items(): axes.plot( possibleParameterValues, v, label=parameterName + ' ' + k )
+		axes.legend()
+	
+	def PlotPredictions( self, truePF=None, figure=None, title='iterations done: {}', addSeparateVisualizerLabel=False ):
+		
+		params = self.parameterOfInterest
+		if isinstance( params, str ): params = params.replace( ',', ' ' ).split()
+		if len( params ) > 1:
+			raise ValueError( 'PlotPredictions() cannot plot when there is more than one parameter of interest' )
+			# If you came here from `Simulate()`, consider preparing your own multidimensional `PsiMarginal`
+			# instance and passing it as the `driver`, instead of passing `parameterOfInterest='alpha,beta'
+			# to the function directly.  Then your `PsiMarginal` can drive and the default one-dimensional
+			# `PsiMarginal` will tag along and call `PlotPredictions()`
+		possibleParameterValues = self.pf.FindParameter( self.parameterOfInterest )[ 1 ].ravel()
+		posterior = self.pf.Distribution( self.pf.LogPosterior(), params=self.parameterOfInterest ).ravel()
+		likelihood = self.pf.Distribution( self.pf.LogScore(), params=self.parameterOfInterest ).ravel()
+		uncertainty = self.entropyHistory[ -1 ]
+		marginalizedPrediction = self.pf.MarginalizedPrediction( self.xCandidates )
+		
+		mle = ModeEstimate( likelihood, possibleParameterValues )
+		map = ModeEstimate( posterior, possibleParameterValues )
+		avg = self.pf.AveragedParameters( self.parameterOfInterest, 'posterior', asDict=False )[ 0 ]
+		
+		data = self.Data()
+		xlim = [ min( self.xCandidates ), max( self.xCandidates ) ]
+		lastXstr = ( '%4g' % data.x.ravel()[ -1 ] ) if data.NumberOfPoints() else 'None'
+		figure, subplots = MakeSubplots( 3, figure=figure, hspace=0.4 )
+		axes = subplots.flat[ 0 ]
+		if axes:
+			axes.plot( self.xCandidates, marginalizedPrediction, label=( self.Label() + ' m' if addSeparateVisualizerLabel else 'M' ) + 'arginalized prediction' )
+			data.Plot( axes=axes, hold=True )
+			if truePF: truePF.Plot( axes=axes, hold=True, xlim=xlim, label='True function' )
+			if data.NumberOfPoints(): axes.plot( data.x[ -1 ], data.y[ -1 ], marker='x', color='r', markersize=15, clip_on=False )
+			axes.set( xlim=xlim, ylim=[ -0.04, 1.04 ], xlabel='x', ylabel='p', title=title.format( data.NumberOfPoints() ) )
+			axes.legend( loc='lower left' if self.xCandidates[ numpy.argmin( marginalizedPrediction ) ] > self.xCandidates[ numpy.argmax( marginalizedPrediction ) ] else 'lower right' )
+		axes = subplots.flat[ 1 ]
+		if axes:
+			try:    hLike = axes.stem( possibleParameterValues, likelihood, label='Likelihood', use_line_collection=True )
+			except: hLike = axes.stem( possibleParameterValues, likelihood, label='Likelihood' )
+			try:    hPost = axes.stem( possibleParameterValues, posterior,  label='Posterior',  use_line_collection=True )
+			except: hPost = axes.stem( possibleParameterValues, posterior,  label='Posterior'  ) # you backward-compatibility-breaking mfs
+			
+			hLike.baseline.set_visible( False ); hPost.baseline.set_visible( False )
+			try:    hLike = [ hLike.markerline ] + list( hLike.stemlines )
+			except: hLike = [ hLike.markerline, hLike.stemlines ]
+			for h in hLike: h.set_color( 'r' ) 
+			if self.parameterOfInterest == 'alpha' and not self.pf.preprocessing: axes.set_xlim( xlim )
+			axes.set( xlabel=self.parameterOfInterest, title=( self.Label() + ' scores -- ' if addSeparateVisualizerLabel else '' ) + 'MLE = %4g; MAP = %4g; AVG = %4g; entropy = %4g' % ( mle, map, avg, uncertainty ) )
+			axes.grid( True )
+			axes.legend()
+		axes = subplots.flat[ 2 ]
+		if axes:
+			axes.plot( self.xCandidates, self.expectedEntropy, 'b:' )
+			axes.plot( self.xCandidates, self.adjustedExpectedEntropy, 'bx-' )  # NaN for "low" stimuli after a miss, if psimarg was initialized with mercy > 0.0
+			pv = self.PendingValue()
+			pv = 'None' if pv is None else '%4g' % pv 
+			axes.set( xlim=xlim, xlabel='potential next x', ylabel='expected entropy (%s)' % self.parameterOfInterest, title=( self.Label() + ' objective function -- ' if addSeparateVisualizerLabel else '' ) + 'last x = %s;  next x = %s' % ( lastXstr, pv ) )
+			axes.grid( True )
+		figure.tight_layout()
+		figure.canvas.draw()
+	
+	@property
+	def entropy( self ):
+		value, = self.pf.Entropy( self.pf.LogPosterior(), params=self.parameterOfInterest ).flat
+		return float( value )
+	
+	def __del__( self ):
+		self.ClosePdfFile()
+		
+	def ClosePdfFile( self ):
+		if self.pdfFile:
+			self.pdfFile.close()
+			self.pdfFile = None
+
+# def Demo() has now been renamed to:
+def Simulate( forcedAlternatives=0, interactive=True, truePF=None, xCandidates=None, shape=None, preprocessing=None, f0=None, start='random', blockSize=1, driver=None, figure=None, gridArgs=(), priorArgs=(), pdfFile=None, **kwargs ):
+	"""
+	`forcedAlternatives` is the number of nAFC alternatives---for subjective designs,
+	pass either 0 or 1 (doesn't matter which). 
+	
+	`interactive` can be False, True, 'go' or 'stop' (True is the same as 'stop').
+	
+	`truePF` if you cannot supply a true `PsychometricFunction` instance, one will be
+	appointed for you by the court.
+	
+	`xCandidates` is the finite set of stimulus intensity values to consider.
+	
+	`start` can be a numeric stimulus intensity value, or it can be `None` (in which
+	case the first stimulus value is the current `.PendingValue()`), or it can be
+	'random'.  Note that 'random' consults the *true* psychometric function to come
+	up with a ballpark of where to start---this tacitly assumes some prior knowledge
+	about the psychometric function. This is basically a quick hack for non-Bayesian
+	methods. If you're using a Bayesian method, prior knowledge would be better
+	expressed by being encoding properly in the priors of the `PsychometricFunction`
+	instance with which the `AdaptiveProcedure` was initialized, and then letting the
+	procedure do its thing naturally with `start=None`.
+
+	`f0` is the operating point of the `PsychometricLinkFunction` instance that will
+	be used for fitting.
+	
+	`driver` is an instance of an AdaptiveProcudure subclass that is responsible for
+	choosing the next stimulus value. If `driver` is omitted (None), then the `PsiMarginal`
+	method is used. Even if a different `driver` is specified, a `PsiMarginal` instance
+	will still run in parallel for visualization purposes (it just will not drive the
+	decisions).
+	
+	`figure` is a `matplotlib.Figure` instance or a positive integer denoting a
+	matplotlib figure number.
+	
+	`**kwargs` are passed forward into the `PsiMarginal` constructor (whether or not
+	`PsiMarginal` is the driver).
+		
+	Return value:  `(driver, psimarg)` (If `PsiMarginal` *was* the driver, these will be
+	two references to the same instance.)
+	"""
+	gridArgs = dict( gridArgs )
+	gridArgs.setdefault( 'alpha',   ( -5,    +5,    101 ) )
+	gridArgs.setdefault( 'beta',    ( -4,    +4,     31 ) )
+	gridArgs.setdefault( 'gamma',   (  0.001, 0.101, 11 ) )
+	gridArgs.setdefault( 'lambda_', (  0.001, 0.101, 11 ) )
+	priorArgs = dict( priorArgs )
+	priorArgs.setdefault( 'alpha',   ( 0.0, 2.0  ) )
+	priorArgs.setdefault( 'beta',    ( 0.0, 1.0  ) )
+	priorArgs.setdefault( 'gamma',   ( 0.0, 0.01 ) )
+	priorArgs.setdefault( 'lambda_', ( 0.0, 1.0  ) ) # let's be pretty agnostic about lambda
+	
+	forcedAlternatives = int( forcedAlternatives )
+	if forcedAlternatives in [ 0, 1 ]:
+		gamma = 0.01
+	else:
+		gamma = 1.0 / forcedAlternatives
+		gridArgs[ 'gamma' ] = None
+		priorArgs[ 'gamma' ] = None
+
+	# with f0 = 0.5, the x=alpha point is around 50% positive for Yes/No, 75% correct for 2AFC, etc. (give or take the effects of gamma and/or lambda)
+	if truePF is None:
+		if shape is None: shape = Logistic
+		if f0 is None: f0 = 0.5 if isinstance( shape, type ) else shape.f0 # ready-made instances will have an .f0
+		if not isinstance( shape, type ): shape = type( shape )
+		truePF = PsychometricFunction( alpha=-2.5, shape=shape( f0 ), preprocessing=preprocessing, gamma=gamma, lambda_=0.01 )
+	
+	if preprocessing is None: preprocessing = truePF.preprocessing
+	if shape is None:         shape         = truePF.shape
+	if f0 is None: f0 = 0.5 if isinstance( shape, type ) else shape.f0
+	if not isinstance( shape, type ): shape = type( shape )
+	
+	if xCandidates is None:
+		xCandidates = numpy.linspace( -5, 5, 51 )
+		xCandidates = Chain( xCandidates, preprocessing, inverse=True )
+	
+	psimarg = PsiMarginal( # this 1-D PsiMarginal will be the visualizer regardless of driver, and is also the default driver
+		parameterOfInterest = 'alpha',
+		xCandidates = xCandidates,
+		blockSize = blockSize, # used in expected entropy calculations (how many new trials to expect)
+		pf = PsychometricFunction(
+			gamma = gamma,
+			shape = shape( f0 ),
+			preprocessing = preprocessing,
+		).Grid( **gridArgs ).SetPrior( **priorArgs ),
+		**kwargs
+	)
+	if driver is None:
+		driver = psimarg
+	elif driver in [ '1D', '1-D' ]:
+		driver = psimarg
+	elif driver in [ '2D', '2-D' ]:
+		driver = PsiMarginal(
+			parameterOfInterest = [ 'alpha', 'beta' ],
+			xCandidates = xCandidates,
+			blockSize = blockSize, # used in expected entropy calculations (how many new trials to expect)
+			pf = PsychometricFunction(
+				shape = shape( f0 ),
+				preprocessing = preprocessing,
+				gamma = gamma,
+			).Grid( **gridArgs ).SetPrior( **priorArgs ),
+			**kwargs
+		)
+	elif isinstance( driver, type ):
+		driver = driver()
+	else:
+		pass # assume `driver` is a ready-made instance
+			
+	if pdfFile and not interactive: raise NotImplementedError( 'for Simulate(), we must be in interactive mode to save a pdf' )
+	# NB: currently there's no provision for saving a `pdfFile` if we're NOT `interactive`.
+	#     While this would be easy to support for simple cases (just pass the filename
+	#     `pdfFile` to the `PsiMarginal` constructor), it would get misleading in cases
+	#     where `Simulate()` is using a separate driver and visualizer (the visualizer
+	#     `psimarg` would be the one to save the figures, and those figures would contain
+	#     no visible acknowledgment that `psigmarg` was not itself the driver; they would
+	#     also not show the truePF). 
+
+	kb = None
+	callback = None
+	if callable( interactive):
+		callback = interactive
+	elif interactive:
+		if pdfFile: pdfFile = PdfMaker.MultipagePdfHandle( pdfFile )
+		figure = ResolveFigure( figure, defaultFigureSizeInches=[ 10, 8.6 ], interactive=True )
+		RaiseFigure()
+		if interactive == True: interactive = 'stop'
+		kb = KeyboardInteractor( state=interactive, figure=figure )
+		def callback( driver ):
+			with PdfMaker( pdfFile, figure=figure ):
+				psimarg.PlotPredictions( truePF=truePF, figure=figure, title=driver.Label() + ' iterations done: {}', addSeparateVisualizerLabel=( driver is not psimarg ) )
+			if driver.Finished(): state, stopping = 'finished', True
+			elif kb and not kb.KeyboardAdvance( waitingStatus='waiting for user...' ): state, stopping = 'aborted', True
+			else: state, stopping = 'computing...', False # TODO: this status could be force-drawn with figure.canvas.flush_events(), but displayed somewhere else so as not to make the title jump around.
+			if figure.axes: UpdateAxesTitle( figure.axes[ 0 ], '{} - ' + state )
+			return stopping
+	driver.Test( truePF, start=start, n=blockSize, callback=callback, parallel=[ psimarg ] )
+	if kb: kb.QToClose()
+	if pdfFile and hasattr( pdfFile, 'close' ): pdfFile.close()
+	
+	visQualifier = ''
+	if driver is not psimarg:
+		print( 'Estimate from driver %s:\n\t%s' % ( driver.Label(), driver.Estimate() ) )
+		visQualifier = 'visualizer '
+	print( 'Estimates from %s%s\n\t%s' % ( visQualifier, psimarg.Label(), psimarg.Estimate( 'all' ) ) )
+	return driver, psimarg