umaudemc 0.13.1__py3-none-any.whl → 0.15.0__py3-none-any.whl

umaudemc/distributed.py

@@ -0,0 +1,322 @@
+#
+# Distributed model checking
+#
+
+import io
+import json
+import os
+import random
+import re
+import selectors
+import socket
+import tarfile
+from array import array
+from contextlib import ExitStack
+
+from .common import load_specification, usermsgs, MaudeFileFinder
+from .statistical import QueryData, check_interval, get_quantile_func, make_parameter_dicts
+
+# Regular expression for Maude inclusions
+LOAD_REGEX = re.compile(rb'^\s*s?load\s+("((?:[^"\\]|\\.)+)"|\S+)')
+EOF_REGEX = re.compile(rb'^\s*eof($|\s)')
+TOP_COMMENT_REGEX = re.compile(rb'(?:\*{3}|-{3})\s*(.)')
+NESTED_COMMENT_REGEX = re.compile(rb'[\(\)]')
+
+
+def strip_comments(fobj):
+	"""Strip Maude comments from a file"""
+
+	comment_depth = 0  # depth of nested multiline comments
+	chunks = []
+
+	for line in fobj:
+		start = 0
+
+		while True:
+			# Inside a multiline comment
+			if comment_depth > 0:
+				# Find a the start or end of a multiline comment
+				if m := NESTED_COMMENT_REGEX.search(line, start):
+					if m.group(0) == b')':  # end
+						comment_depth -= 1
+
+					else:  # start
+						comment_depth += 1
+
+					start = m.end()
+
+				# The whole line is inside a multiline comment
+				else:
+					break
+
+			# Not inside a comment
+			else:
+				if m := TOP_COMMENT_REGEX.search(line, start):
+					chunks.append(line[start:m.start()] + b'\n')
+
+					# Start of multiline comment
+					if m.group(1) == b'(':
+						start = m.end()
+						comment_depth += 1
+
+					# Single line comment
+					else:
+						break
+
+				# No comment at all
+				else:
+					chunks.append(line[start:])
+					break
+
+		if chunks:
+			yield b' '.join(chunks)
+			chunks.clear()
+
+
+def flatten_maude_file(filename, fobj):
+	"""Scan sources for its dependencies"""
+
+	# Maude file finder
+	mfinder = MaudeFileFinder()
+
+	# Files are explored depth-first
+	stack = [(lambda v: (filename, v, strip_comments(v)))(open(filename, 'rb'))]
+	# Files already included to avoid double inclusion (loads are interpreted as sloads)
+	seen = {os.path.realpath(filename)}
+
+	while stack:
+		fname, file, lines = stack[-1]
+
+		while line := next(lines, None):
+			# Load commands are only allowed at the beginning of a line
+			if m := LOAD_REGEX.match(line):
+				# File name (with quotes, maybe)
+				next_fname = m.group(1).decode()
+
+				if next_fname[0] == '"':
+					next_fname = next_fname[1:-1].replace(r'\"', '"')
+
+				# Inclusions are tricky
+				next_path, is_std = mfinder.find(next_fname, os.path.dirname(fname))
+
+				# For standard files, we preserve the inclusion
+				if is_std:
+					fobj.write(line)
+
+				# Otherwise, we copy the file unless already done
+				elif next_path not in seen:
+					next_file = open(next_path, 'rb')
+					stack.append((next_path, next_file, strip_comments(next_file)))
+					seen.add(next_path)
+					break
+
+			elif EOF_REGEX.match(line):
+				line = None
+				break
+			else:
+				fobj.write(line)
+
+		# Whether the file is exhausted
+		if line is None:
+			fobj.write(b'\n')  # just in case there is no line break at end of file
+			stack.pop()
+			file.close()
+
+
+def process_dspec(dspec, fname):
+	"""Normalize a distributed SMC specification"""
+
+	# Normalize workers to a dictionary
+	workers = dspec.get('workers')
+
+	if not isinstance(workers, list):
+		usermsgs.print_error_file('the distribution specification does not contain a list-valued \'workers\' key.', fname)
+		return False
+
+	for k, worker in enumerate(workers):
+		# Strings address:port are allowed
+		if isinstance(worker, str):
+			try:
+				address, port = worker.split(':')
+				worker = {'address': address, 'port': int(port)}
+
+			except ValueError:
+				usermsgs.print_error_file(f'bad address specification {worker} for worker {k + 1} '
+				                          '(it should be <address>:<port>).', fname)
+				return False
+
+			workers[k] = worker
+
+		# Otherwise, it must be a dictionary
+		elif not isinstance(worker, dict):
+			usermsgs.print_error_file(f'the specification for worker {k + 1} is not a dictionary.', fname)
+			return False
+
+		# With address and port keys
+		else:
+			for key, ktype in (('address', str), ('port', int)):
+				if key not in worker:
+					usermsgs.print_error_file(f'missing key \'{key}\' for worker {k + 1}.', fname)
+					return False
+
+				if not isinstance(worker[key], ktype):
+					usermsgs.print_error_file(f'wrong type for key \'{key}\' in worker {k + 1}, {ktype.__name__} expected.', fname)
+					return False
+
+		# Name just for reference in errors and messages
+		if 'name' not in worker:
+			worker['name'] = f'{worker["address"]}:{worker["port"]}'
+
+	return True
+
+
+def setup_workers(args, initial_data, dspec, seen_files, stack):
+	"""Setup workers and send problem data"""
+
+	workers = dspec['workers']
+
+	# Generate a random seed for each worker
+	seeds = [random.getrandbits(20) for _ in range(len(workers))]
+
+	# Data to be passed to the external machine
+	COPY = ('initial', 'strategy', 'module', 'metamodule', 'opaque', 'full_matchrew',
+	        'purge_fails', 'merge_states', 'assign', 'block', 'query', 'assign', 'advise', 'verbose')
+
+	data = {key: args.__dict__[key] for key in COPY} | {'file': 'source.maude'}
+
+	# Make a flattened version of the Maude file
+	flat_source = io.BytesIO()
+	flatten_maude_file(initial_data.filename, flat_source)
+
+	flat_info = tarfile.TarInfo('source.maude')
+	flat_info.size = flat_source.getbuffer().nbytes
+
+	# Save the sockets for each worker
+	sockets = []
+
+	for worker, seed in zip(workers, seeds):
+		address, port = worker['address'], worker['port']
+
+		try:
+			sock = socket.create_connection((address, int(port)))
+
+		except ConnectionRefusedError:
+			usermsgs.print_error(f'Connection refused by worker \'{worker["name"]}\'.')
+			return None
+
+		stack.enter_context(sock)
+		sockets.append(sock)
+
+		# Send the input data
+		input_data = data | {'seed': seed}
+
+		if block_size := worker.get('block'):
+			input_data['block'] = block_size  # if specified
+
+		input_data = json.dumps(input_data).encode()
+		sock.sendall(len(input_data).to_bytes(4) + input_data)
+
+		# Send the relevant files
+		with sock.makefile('wb', buffering=0) as fobj:
+			with tarfile.open(mode='w|gz', fileobj=fobj) as tarf:
+				flat_source.seek(0)
+				tarf.addfile(flat_info, flat_source)
+
+				for file in seen_files:
+					relpath = os.path.relpath(file)
+
+					if relpath.startswith('..'):
+						usermsgs.print_error('QuaTEx file outside the working tree, it will not be included and the execution will fail.')
+					else:
+						tarf.add(relpath)
+
+			fobj.flush()
+
+		# Receive confirmation from the remote
+		answer = sock.recv(1)
+
+		if answer != b'o':
+			usermsgs.print_error(f'Configuration error in {worker["name"]} worker.')
+			return None
+
+	return sockets
+
+
+def distributed_check(args, initial_data, min_sim, max_sim, program, seen_files):
+	"""Distributed statistical model checking"""
+
+	# Load the distribution specification
+	if (dspec := load_specification(args.distribute, 'distribution specification')) is None \
+	   or not process_dspec(dspec, args.distribute):
+		return None, None
+
+	# Gather all sockets in a context to close them when we finish
+	with ExitStack() as stack:
+
+		# Socket to connect with the workers
+		if not (sockets := setup_workers(args, initial_data, dspec, seen_files, stack)):
+			return None, None
+
+		print('All workers are ready. Starting...')
+
+		# Use a selector to wait for updates from any worker
+		selector = selectors.DefaultSelector()
+
+		for sock, data in zip(sockets, dspec['workers']):
+			selector.register(sock, selectors.EVENT_READ, data=data)
+			sock.send(b'c')
+
+		buffer = array('d')
+
+		# Query data
+		qdata = [QueryData(k, idict)
+		         for k, qinfo in enumerate(program.query_locations)
+		         for idict in make_parameter_dicts(qinfo[3])]
+		nqueries = len(qdata)
+		num_sims = 0
+
+		quantile = get_quantile_func()
+
+		while sockets:
+			events = selector.select()
+			finished = []
+
+			for key, _ in events:
+				sock = key.fileobj
+
+				answer = sock.recv(1)
+
+				if answer == b'b':
+					data = sock.recv(16 * nqueries)
+					buffer.frombytes(data)
+
+					for k in range(nqueries):
+						qdata[k].sum += buffer[k]
+						qdata[k].sum_sq += buffer[nqueries + k]
+
+					num_sims += key.data['block']
+
+					del buffer[:]
+					finished.append(key.fileobj)
+
+				else:
+					usermsgs.print_error(f'Server {key.data["name"]} disconnected or misbehaving')
+					selector.unregister(key.fileobj)
+					sockets.remove(key.fileobj)
+
+			# Check whether the simulation has converged
+			converged = check_interval(qdata, num_sims, args.alpha, args.delta, quantile, args.verbose)
+
+			if converged or max_sim and num_sims >= max_sim:
+				break
+
+			for sock in finished:
+				sock.send(b'c')
+
+			finished.clear()
+
+		# Send stop signal to all workers
+		for sock in sockets:
+			sock.send(b's')
+
+	return num_sims, qdata

umaudemc/formatter.py

@@ -19,7 +19,7 @@ def apply_state_format(graph, index, sformat, terms, use_term=False, use_strat=F
 	:type sformat: str
 	:param terms: Term patterns to appear in the format string.
 	:type terms: list of string
-	:param use_term: Whether the term is actually used (only for effiency).
+	:param use_term: Whether the term is actually used (only for efficiency).
 	:type use_term: bool
 	:param use_strat: Whether the strategy is actually used (only for effiency).
 	:type use_strat: bool

umaudemc/kleene.py

@@ -13,6 +13,10 @@ class KleeneExecutionState(GraphExecutionState):
 		super().__init__(term, pc, stack, conditional, graph_node,
 		                 (extra, set() if iterations is None else iterations))
 
+		# The second position of the extra attribute is a set of iteration tags of the
+		# form ((pc, ctx), enters) where (pc, ctx) identifies the iteration and enters
+		# indicates whether it enters or leaves
+
 	def copy(self, term=None, pc=None, stack=None, conditional=False, graph_node=None, extra=None):
 		"""Clone state with possibly some changes"""
 
@@ -44,6 +48,7 @@ class KleeneRunner(GraphRunner):
 
 	def kleene(self, args, stack):
 		"""Keep track of iterations"""
+		# Identify each dynamic context with a number
 		context_id = self.iter_contexts.setdefault(self.current_state.stack, len(self.iter_contexts))
 		self.current_state.add_kleene(((args[0], context_id), args[1]))
 		super().kleene(args, stack)
@@ -119,7 +124,7 @@ class StrategyKleeneGraph:
 	def expand(self):
 		"""Expand the underlying graph"""
 
-		for k, state in enumerate(self.state_list):
+		for state in self.state_list:
 			for child in state.children:
 				if child not in self.state_map:
 					self.state_map[child] = len(self.state_list)

umaudemc/opsem.py

@@ -65,7 +65,7 @@ class OpSemInstance:
 
 	@classmethod
 	def get_instantiation(cls, module, metamodule=None, semantics_module=None, preds_module=None):
-		"""Get the Maude code to instantiate the sematnics for the given problem"""
+		"""Get the Maude code to instantiate the semantics for the given problem"""
 
 		# Use the default semantics and predicates module if not specified
 		if semantics_module is None:

umaudemc/probabilistic.py

@@ -714,7 +714,7 @@ class GeneralizedMetadataGraph:
 			edges = []
 
 			for child, subs, ctx, rl in term.apply(None):
-				# Term.apply do not reduce the terms itself
+				# Term.apply does not reduce the terms itself
 				self.rewrite_count += 1 + child.reduce()
 				index = self.term_map.get(child)
 

umaudemc/pyslang.py

@@ -30,7 +30,7 @@ class Instruction:
 	RWCSTART = 11  # list of (pattern, condition, starting pattern)
 	RWCNEXT = 12  # list of (end pattern, condition, starting pattern/right-hand side)
 	NOTIFY = 13  # list of lists of variables of the nested pending matchrew subterms
-	SAMPLE = 14  # (variable, distribution callable, its arguments)
+	SAMPLE = 14  # (variable, distribution sampler, distribution generator, its arguments)
 	WMATCHREW = 15  # like MATCHREW + (weight)
 	ONE = 16  # no arguments
 	CHECKPOINT = 17  # no arguments
@@ -130,12 +130,30 @@ def merge_substitutions(sb1, sb2):
 	return maude.Substitution({**dict(sb1), **dict(sb2)})
 
 
-def bernoulli_distrib(p):
-	"""Bernoulli distribution"""
+def bernoulli_sampler(p):
+	"""Bernoulli distribution sampler"""
 
 	return 1.0 if random.random() < p else 0.0
 
 
+def bernoulli_distribution(p):
+	"""Explicit Bernoulli distribution"""
+
+	return (p, 1.0), (1 - p, 0.0)
+
+
+def uniform_sampler(start, end):
+	"""Integer uniform distribution sampler"""
+
+	return random.randint(start, end + 1)
+
+
+def uniform_distribution(start, end):
+	"""Explicit uniform distribution"""
+
+	return ((1, k) for k in range(start, end + 1))
+
+
 def find_cycles(initial, next_fn):
 	"""Find cycles in the graph and return the points where to cut them"""
 
@@ -290,6 +308,9 @@ class StratCompiler:
 		self.mtc_symbol = ml.findSymbol('_:=_', (term_kind, term_kind), condition_kind)
 		self.stc_symbol = ml.findSymbol('_:_', (term_kind, term_kind), condition_kind)
 
+		# Kind of natural numbers (if any)
+		self.nat_kind = self.get_special_kind('ACU_NumberOpSymbol')
+
 		# Program counters of the generated strategy calls
 		self.calls = []
 
@@ -311,6 +332,13 @@ class StratCompiler:
 
 		return list(term.arguments())
 
+	def get_special_kind(self, hook):
+		"""Get a special kind by the hook name of one of its symbols"""
+
+		for s in self.m.getSymbols():
+			if (id_hooks := s.getIdHooks()) and id_hooks[0][0] == hook:
+				return s.getRangeSort().kind()
+
 	def substitution2dict(self, substitution):
 		"""Convert a metalevel substitution to a dictionary"""
 
@@ -670,15 +698,26 @@ class StratCompiler:
 		name = str(name)[1:]
 		args = tuple(map(self.m.downTerm, self.get_list(args, self.empty_term_list, self.term_list_symbol)))
 
-		name = {
-			'bernoulli': bernoulli_distrib,
-			'uniform': random.uniform,
+		# Whether the distribution is discrete
+		discrete = name == 'bernoulli' or (name == 'uniform' and self.nat_kind is not None
+			and all(arg.getSort().kind() == self.nat_kind for arg in args))
+
+		# Sampler to obtain one value from the distribution
+		sampler = {
+			'bernoulli': bernoulli_sampler,
+			'uniform': uniform_sampler if discrete else random.uniform,
 			'exp': random.expovariate,
 			'norm': random.normalvariate,
 			'gamma': random.gammavariate,
 		}[name]
 
-		p.append(Instruction.SAMPLE, (self.m.downTerm(variable), name, args))
+		# Full distribution generator (for graph generation in discrete distributions)
+		generator = None
+
+		if discrete:
+			generator = bernoulli_distribution if name == 'bernoulli' else uniform_distribution
+
+		p.append(Instruction.SAMPLE, (self.m.downTerm(variable), sampler, generator, args))
 		self.generate(strat, p, tail)
 		p.append(Instruction.POP)
 
@@ -1478,6 +1517,9 @@ class StratRunner:
 		elif stack.pc:
 			self.current_state.pc = stack.pc
 
+		else:
+			self.current_state.pc += 1
+
 		# Pop the stack node
 		if stack.parent:
 			self.current_state.stack = self.current_state.stack.parent
@@ -1835,10 +1877,10 @@ class StratRunner:
 
 		self.next_pending()
 
-	def sample(self, args, stack):
-		"""Sample a probability distribution into a variable"""
+	def sample_get_params(self, args, stack):
+		"""Extract distribution parameters from a sample call"""
 
-		variable, dist, dargs = args
+		_, _, generator, dargs = args
 
 		# Arguments of the distribution instantiated in the variable context
 		dargs = [(stack.venv.instantiate(arg) if stack.venv else arg) for arg in dargs]
@@ -1847,9 +1889,20 @@ class StratRunner:
 		for arg in dargs:
 			self.nr_rewrites += arg.reduce()
 
-		dargs = [float(arg) for arg in dargs]
+		# Argument type to convert arguments
+		atype = float if generator is not uniform_distribution else int
+
+		return [atype(arg) for arg in dargs]
+
+	def sample(self, args, stack):
+		"""Sample a probability distribution into a variable"""
+
+		variable, sampler, _, dargs = args
+
+		dargs = self.sample_get_params(args, stack)
 
-		new_variable = {variable: variable.symbol().getModule().parseTerm(str(dist(*dargs)))}
+		module = variable.symbol().getModule()
+		new_variable = {variable: module.parseTerm(str(sampler(*dargs)))}
 
 		self.current_state.pc += 1
 		self.current_state.stack = StackNode(parent=self.current_state.stack,
@@ -1963,7 +2016,7 @@ class GraphState:
 
 
 class BadProbStrategy(Exception):
-	"""Bad probabilitistic strategy that cannot engender MDPs or DTMCs"""
+	"""Bad probabilistic strategy that cannot engender MDPs or DTMCs"""
 
 	def __init__(self):
 		super().__init__('Strategies are not allowed to make nondeterministic choices '
@@ -2001,7 +2054,7 @@ class GraphRunner(StratRunner):
 
 		# Solution states, indexed by term (we introduce them as children
 		# of graph states where both a solution of the strategy and a
-		# different sucessor can be reached)
+		# different successor can be reached)
 		self.solution_states = {}
 
 	def get_solution_state(self, term):
@@ -2081,9 +2134,12 @@ class GraphRunner(StratRunner):
 		"""Return from a strategy call or similar construct"""
 
 		# Return from a strategy call
-		if stack.pc:
+		if stack.pc is not None:
 			self.current_state.pc = stack.pc
 
+		else:
+			self.current_state.pc += 1
+
 		# Actually pop the stack node
 		if stack.parent:
 			self.current_state.stack = self.current_state.stack.parent
@@ -2189,6 +2245,27 @@ class GraphRunner(StratRunner):
 
 		self.next_pending()
 
+	def sample(self, args, stack):
+		variable, sampler, generator, dargs = args
+
+		# Only special behavior for discrete distributions
+		if not generator:
+			return super().sample(args, stack)
+
+		# Arguments of the distribution instantiated in the variable context
+		dargs = self.sample_get_params(args, stack)
+
+		# Generate a pending state for each possible outcome
+		module = variable.symbol().getModule()
+
+		for _, value in generator(*dargs):
+			self.pending.append(self.current_state.copy(stack=StackNode(
+				parent=self.current_state.stack,
+				venv=merge_substitutions(stack.venv, {variable: module.parseTerm(str(value))})))
+			)
+
+		self.next_pending()
+
 	def run(self):
 		"""Run the strategy to generate the graph"""
 
@@ -2298,20 +2375,8 @@ class MarkovRunner(GraphRunner):
 
 		graph_state.child_choices = tuple(new_choices)
 
-	def choice(self, args, stack):
-		"""A choice node"""
-
-		weights, targets = zip(*args)
-		weights = list(self.compute_weights(weights, stack.venv))
-
-		# Remove options with null weight
-		targets = [target for k, target in enumerate(targets) if weights[k] != 0.0]
-		weights = [w for w in weights if w != 0.0]
-
-		# If there is only a positive weight, we can proceed with it
-		if len(weights) == 1:
-			self.current_state.pc = targets[0]
-			return
+	def spawn_choice(self, weights, new_xss):
+		"""Spawn tasks for a choice"""
 
 		# Otherwise, if there is at least one
 		if weights:
@@ -2323,8 +2388,7 @@ class MarkovRunner(GraphRunner):
 			# These graph states cannot be pushed to the DFS stack (because we
 			# are not exploring them yet), they should be pushed when the
 			# execution state new_xs is executed, so we add them to push_state
-			for k, target in enumerate(targets):
-				new_xs = self.current_state.copy(pc=target)
+			for k, new_xs in enumerate(new_xss):
 				self.pending.append(new_xs)
 				self.push_state[new_xs] = new_states[k]
 
@@ -2333,6 +2397,26 @@ class MarkovRunner(GraphRunner):
 
 		self.next_pending()
 
+	def choice(self, args, stack):
+		"""A choice node"""
+
+		weights, targets = zip(*args)
+		weights = list(self.compute_weights(weights, stack.venv))
+
+		# Remove options with null weight
+		targets = [target for k, target in enumerate(targets) if weights[k] != 0.0]
+		weights = [w for w in weights if w != 0.0]
+
+		# If there is only a positive weight, we can proceed with it
+		if len(weights) == 1:
+			self.current_state.pc = targets[0]
+			return
+
+		# New execution states
+		new_xss = (self.current_state.copy(pc=target) for target in targets)
+
+		self.spawn_choice(weights, new_xss)
+
 	def wmatchrew(self, args, stack):
 		"""A matchrew with weight node"""
 
@@ -2376,24 +2460,34 @@ class MarkovRunner(GraphRunner):
 			self.current_state = new_xss[0]
 			return
 
-		# Otherwise, if there is at least one
-		if weights:
-			graph_state = self.dfs_stack[-1]
+		self.spawn_choice(weights, new_xss)
 
-			# Create a new graph state (without term) for each branch of the matchrew
-			new_states = [self.GraphState(None) for _ in range(len(weights))]
+	def sample(self, args, stack):
+		variable, sampler, generator, dargs = args
 
-			# These graph states cannot be pushed to the DFS stack (because we
-			# are not exploring them yet), they should be pushed when the
-			# execution state new_xss[k] is executed, so we add them to push_state
-			for k in range(len(targets)):
-				self.pending.append(new_xss[k])
-				self.push_state[new_xss[k]] = new_states[k]
+		# Only a special behavior for discrete distributions
+		if not generator:
+			return super().sample(args, stack)
 
-			# Add the resolved choice to the graph state
-			graph_state.child_choices.add(tuple(zip(weights, new_states)))
+		# Arguments of the distribution instantiated in the variable context
+		dargs = self.sample_get_params(args, stack)
 
-		self.next_pending()
+		# Generate the distribution of options
+		module = variable.symbol().getModule()
+		weights, new_xss = [], []
+
+		for weight, value in generator(*dargs):
+			weights.append(weight)
+			new_xss.append(self.current_state.copy(stack=StackNode(
+				parent=self.current_state.stack,
+				venv=merge_substitutions(stack.venv, {variable: module.parseTerm(str(value))})
+			)))
+
+		# If there is only a positive weight, we can proceed with it
+		if len(weights) == 1:
+			self.current_state = new_xss[0]
+
+		self.spawn_choice(weights, new_xss)
 
 	def notify(self, args, stack):
 		"""Record a transition in the graph"""
@@ -2594,6 +2688,9 @@ class RandomRunner(StratRunner):
 		if stack.pc:
 			self.current_state.pc = stack.pc
 
+		else:
+			self.current_state.pc += 1
+
 		# Pop the stack node
 		if stack.parent:
 			self.current_state.stack = self.current_state.stack.parent