nemtus-catparser 3.2.0__py3-none-any.whl

catparser/AstPostProcessor.py

@@ -0,0 +1,97 @@
+from .ast import AstException, Struct, StructInlinePlaceholder
+
+
+class AstPostProcessor:
+	"""Post processes AST type descriptors."""
+
+	def __init__(self, type_descriptors):
+		self.raw_type_descriptors = type_descriptors
+		self.type_descriptor_map = {model.name: model for model in self.raw_type_descriptors}
+
+	@property
+	def type_descriptors(self):
+		# filter out inline structs
+		return [model for model in self.raw_type_descriptors if not hasattr(model, 'disposition') or 'inline' != model.disposition]
+
+	def apply_attributes(self):
+		"""Sets properties from attributes within all structures."""
+		for model in self._structs():
+			for field in model.fields:
+				if not hasattr(field, 'attributes') or not field.attributes:
+					continue
+
+				for attribute in field.attributes:
+					if not hasattr(field.field_type, attribute.name):
+						raise AstException(f'field {field.name} ({field.field_type}) does not have property {attribute.name}')
+
+					setattr(field.field_type, attribute.name, attribute.values)
+
+	def _structs(self):
+		return [model for _, model in self.type_descriptor_map.items() if isinstance(model, Struct)]
+
+	def expand_named_inlines(self):
+		"""Expands named inline fields within all structures."""
+
+		for model in self._structs_with_named_inlines():
+			original_fields = model.fields[:]
+			model.fields = []
+
+			for field in original_fields:
+				if not self._is_named_inline(field):
+					model.fields.append(field)
+					continue
+
+				if field.field_type not in self.type_descriptor_map:
+					raise AstException(f'struct {model.name} contains named inline of unknown type {field.field_type}')
+
+				referenced_type_model = self.type_descriptor_map[field.field_type]
+				model.fields.extend(referenced_type_model.apply_inline_template(field))
+
+	def _structs_with_named_inlines(self):
+		return [
+			model for _, model in self.type_descriptor_map.items()
+			if isinstance(model, Struct) and any(self._is_named_inline(field) for field in model.fields)
+		]
+
+	@staticmethod
+	def _is_named_inline(field):
+		return hasattr(field, 'disposition') and 'inline' == field.disposition and hasattr(field, 'name')
+
+	def expand_unnamed_inlines(self):
+		"""Expands unnamed inline fields within all structures."""
+
+		for model in self._structs_with_unnamed_inlines():
+			while self._has_unnamed_inline_field(model):
+				original_fields = model.fields[:]
+				model.fields = []
+
+				for field in original_fields:
+					if not isinstance(field, StructInlinePlaceholder):
+						model.fields.append(field)
+						continue
+
+					if field.inlined_typename not in self.type_descriptor_map:
+						raise AstException(f'struct {model.name} contains unnamed inline of unknown type {field.inlined_typename}')
+
+					referenced_type_model = self.type_descriptor_map[field.inlined_typename]
+					if 'abstract' == referenced_type_model.disposition:
+						model.factory_type = referenced_type_model.name
+					elif referenced_type_model.factory_type:
+						model.factory_type = referenced_type_model.factory_type
+
+					model.fields.extend(referenced_type_model.fields)
+					if referenced_type_model.attributes:
+						if not model.attributes:
+							model.attributes = []
+
+						model.attributes.extend(referenced_type_model.attributes)
+
+	def _structs_with_unnamed_inlines(self):
+		return [
+			model for _, model in self.type_descriptor_map.items()
+			if self._has_unnamed_inline_field(model)
+		]
+
+	@staticmethod
+	def _has_unnamed_inline_field(model):
+		return isinstance(model, Struct) and any(isinstance(field, StructInlinePlaceholder) for field in model.fields)

catparser/AstValidator.py

@@ -0,0 +1,243 @@
+from enum import Enum
+
+from .ast import Array, Conditional, Enum, FixedSizeInteger, Struct, StructInlinePlaceholder
+
+
+class ErrorDescriptor:
+	"""Describes an AST validation error."""
+
+	def __init__(self, message, typename, field_names=None):
+		self.message = message
+		self.typename = typename
+		self.field_names = [field_names] if isinstance(field_names, str) else field_names
+
+	def __eq__(self, rhs):
+		return isinstance(rhs, ErrorDescriptor) and str(self) == str(rhs)
+
+	def __repr__(self):
+		if self.field_names:
+			joined_field_names = ', '.join(self.field_names)
+			return f'[{self.typename}::{{ {joined_field_names} }}] {self.message}'
+
+		return f'[{self.typename}] {self.message}'
+
+
+class AstValidator:
+	"""Validates AST type descriptors."""
+
+	class Mode(Enum):
+		"""Validation mode."""
+
+		PRE_EXPANSION = 1
+		POST_EXPANSION = 2
+
+	def __init__(self, type_descriptors):
+		self.raw_type_descriptors = type_descriptors
+		self.type_descriptor_map = {model.name: model for model in self.raw_type_descriptors}
+
+		self.mode = self.Mode.PRE_EXPANSION
+		self.errors = []
+
+	def set_validation_mode(self, mode):
+		"""Sets the validation mode."""
+		self.mode = mode
+
+	def validate(self):
+		"""Validates all types for correctness."""
+
+		for _, model in self.type_descriptor_map.items():
+			if isinstance(model, Enum):
+				self._validate_enum(model)
+
+			if isinstance(model, Struct):
+				self._validate_struct(model)
+
+	def _validate_enum(self, model):
+		duplicate_names = self._find_duplicate_names(model.values)
+		if duplicate_names:
+			self.errors.append(ErrorDescriptor('duplicate enum values', model.name, duplicate_names))
+
+	def _validate_struct(self, model):
+		duplicate_names = self._find_duplicate_names(model.fields)
+		if duplicate_names:
+			self.errors.append(ErrorDescriptor('duplicate struct fields', model.name, duplicate_names))
+
+		field_map = {field.name: field for field in model.fields if hasattr(field, 'name')}
+		for field in model.fields:
+			def create_error_descriptor(message, error_field=field):
+				return ErrorDescriptor(message, model.name, error_field.name)
+
+			if isinstance(field, StructInlinePlaceholder):
+				self._validate_unnamed_inline(field, lambda message: ErrorDescriptor(message, model.name))
+			else:
+				self._validate_struct_field(field, field_map, create_error_descriptor)
+
+		if self.Mode.PRE_EXPANSION != self.mode:
+			self._check_struct_attributes(model, field_map)
+
+	def _validate_unnamed_inline(self, field, create_error_descriptor):
+		if not self._is_known_type(field.inlined_typename):
+			self.errors.append(create_error_descriptor(f'reference to unknown inlined type "{field.inlined_typename}"'))
+		else:
+			# all dispositions are allowed as unnamed inline
+			pass
+
+	def _validate_struct_field(self, field, field_map, create_error_descriptor):
+		if not self._is_known_type(field.field_type):
+			self.errors.append(create_error_descriptor(f'reference to unknown type "{field.field_type}"'))
+		else:
+			if 'inline' == field.disposition and 'inline' != self.type_descriptor_map[field.field_type].disposition:
+				self.errors.append(create_error_descriptor(f'named inline field referencing non inline struct "{field.field_type}"'))
+
+		if isinstance(field.field_type, FixedSizeInteger):
+			self._validate_integer(field.field_type, field_map, create_error_descriptor)
+
+		if isinstance(field.field_type, Array):
+			self._validate_array(field.field_type, field_map, create_error_descriptor)
+
+		if field.value is not None:
+			if 'sizeof' == field.disposition:
+				self._validate_sizeof(field, field_map, create_error_descriptor)
+			elif isinstance(field.value, Conditional):
+				self._validate_conditional(field, field_map, create_error_descriptor)
+			else:
+				self._validate_in_range(field.field_type, field.value, create_error_descriptor)
+
+		if field.attributes:
+			for attribute in field.attributes:
+				if not hasattr(field.field_type, attribute.name):
+					self.errors.append(create_error_descriptor(f'inapplicable attribute "{attribute.name}"'))
+
+	def _validate_integer(self, field_type, field_map, create_error_descriptor):
+		sizeref = field_type.sizeref
+		if not sizeref:
+			return
+
+		if sizeref.property_name not in field_map:
+			self.errors.append(create_error_descriptor(f'reference to unknown sizeref property "{sizeref.property_name}"'))
+
+	def _validate_array(self, field_type, field_map, create_error_descriptor):
+		element_type = field_type.element_type
+		is_sort_key_valid = True
+		sort_key = field_type.sort_key
+		if not self._is_known_type(element_type):
+			self.errors.append(create_error_descriptor(f'reference to unknown element type "{element_type}"'))
+			is_sort_key_valid = not sort_key
+		else:
+			is_sort_key_valid = not sort_key or any(
+				sort_key == element_field.name for element_field in self.type_descriptor_map[element_type].fields
+			)
+
+		if not is_sort_key_valid:
+			self.errors.append(create_error_descriptor(f'reference to unknown sort_key property "{sort_key}"'))
+
+		size = field_type.size
+		if isinstance(size, str) and size not in field_map:
+			self.errors.append(create_error_descriptor(f'reference to unknown size property "{size}"'))
+
+	def _validate_sizeof(self, field, field_map, create_error_descriptor):
+		if field.value not in field_map:
+			self.errors.append(create_error_descriptor(f'reference to unknown sizeof property "{field.value}"'))
+			return
+
+		reference_typename = field_map[field.value].field_type
+		reference_type = None if not isinstance(reference_typename, str) else self.type_descriptor_map[reference_typename]
+		if not isinstance(reference_type, Struct):
+			self.errors.append(create_error_descriptor(f'sizeof property references fixed size type "{reference_typename}"'))
+		elif not reference_type.is_size_implicit:
+			message = f'sizeof property references type "{reference_typename}" without is_size_implicit attribute'
+			self.errors.append(create_error_descriptor(message))
+
+	def _validate_conditional(self, field, field_map, create_error_descriptor):
+		linked_field_name = field.value.linked_field_name
+
+		if linked_field_name not in field_map:
+			self.errors.append(create_error_descriptor(f'reference to unknown condition field "{linked_field_name}"'))
+		else:
+			self._validate_in_range(field_map[linked_field_name].field_type, field.value.value, create_error_descriptor)
+
+	def _validate_in_range(self, value_type, value, create_error_descriptor):
+		if isinstance(value_type, str):
+			if self._is_known_type(value_type):
+				value_type = self.type_descriptor_map[value_type]
+
+				if isinstance(value_type, Enum):
+					if not any(value == enum_value.name for enum_value in value_type.values):
+						self.errors.append(create_error_descriptor(f'field value "{value}" is not a valid enum value'))
+
+					return
+
+		if not isinstance(value, int):
+			self.errors.append(create_error_descriptor(f'field value "{value}" is not a valid numeric value'))
+
+	def _check_struct_attributes(self, model, field_map):
+		if self._check_known_field(model, field_map, 'size'):
+			if not isinstance(field_map[model.size].field_type, FixedSizeInteger):
+				self.errors.append(ErrorDescriptor(f'reference to "size" property "{model.size}" has unexpected type', model.name))
+
+		self._check_known_field(model, field_map, 'discriminator', True)
+
+		self._check_comparer(model, field_map)
+		self._check_initializers(model, field_map)
+
+	def _check_comparer(self, model, field_map):
+		if not model.comparer:
+			return
+
+		for (property_name, transform) in model.comparer:
+			if property_name not in field_map:
+				self.errors.append(ErrorDescriptor(f'reference to unknown "comparer" property "{property_name}"', model.name))
+			if transform not in (None, 'ripemd_keccak_256'):
+				self.errors.append(ErrorDescriptor(f'reference to unknown "comparer" transform "{transform}"', model.name))
+
+	def _check_initializers(self, model, field_map):
+		if not model.initializers:
+			return
+
+		for initializer in model.initializers:
+			is_valid = True
+			is_concrete = model.disposition not in ('abstract', 'inline')
+			for property_name, raise_error in [(initializer.target_property_name, True), (initializer.value, is_concrete)]:
+				if property_name not in field_map:
+					is_valid = False
+					if raise_error:
+						self.errors.append(ErrorDescriptor(
+							f'reference to unknown "intializes" property "{property_name}"',
+							model.name))
+
+			if not is_valid:
+				continue
+
+			# str compare is good enough as it lets us differentiate among subtypes of things like FixedSizeInteger
+			if str(field_map[initializer.target_property_name].field_type) != str(field_map[initializer.value].field_type):
+				self.errors.append(ErrorDescriptor(
+					f'property "{initializer.target_property_name}" has initializer "{initializer.value}" of different type',
+					model.name))
+
+	def _check_known_field(self, model, field_map, property_name, multi_value=False):
+		values = getattr(model, property_name)
+		if not values:
+			return False
+
+		if not multi_value:
+			values = [values]
+
+		has_error = False
+		for value in values:
+			if value not in field_map:
+				self.errors.append(ErrorDescriptor(f'reference to unknown "{property_name}" property "{value}"', model.name))
+				has_error = True
+
+		return not has_error
+
+	def _is_known_type(self, typename):
+		return not isinstance(typename, str) or typename in self.type_descriptor_map
+
+	@staticmethod
+	def _find_duplicate_names(items):
+		unique_names = set()
+		duplicate_names = set()
+		for item in filter(lambda item: hasattr(item, 'name'), items):
+			(unique_names if item.name not in unique_names else duplicate_names).add(item.name)
+
+		return duplicate_names

catparser/CatsLarkParser.py

@@ -0,0 +1,187 @@
+
+from lark import Lark, Transformer
+from lark.indenter import Indenter
+
+from .ast import (
+	Alias,
+	Array,
+	Attribute,
+	Comment,
+	Conditional,
+	Enum,
+	EnumValue,
+	FixedSizeBuffer,
+	FixedSizeInteger,
+	Struct,
+	StructField,
+	StructInlinePlaceholder
+)
+
+
+def create_cats_lark_parser():
+	"""Creates a CATS grammar-based Lark parser that can be used for parsing CATS files."""
+
+	class CatbufferIndenter(Indenter):
+		NL_type = '_NL'
+		OPEN_PAREN_types = []
+		CLOSE_PAREN_types = []
+		INDENT_type = '_INDENT'
+		DEDENT_type = '_DEDENT'
+		tab_len = 4  # pylint: disable=invalid-name
+
+	class CatbufferTransformer(Transformer):
+		# pylint: disable=too-many-public-methods
+
+		# region PODs
+
+		@staticmethod
+		def ESCAPED_STRING(string):  # pylint: disable=invalid-name
+			return string[1:-1]  # trim quotes
+
+		@staticmethod
+		def DEC_NUMBER(string):  # pylint: disable=invalid-name
+			return int(string, 10)
+
+		@staticmethod
+		def HEX_NUMBER(string):  # pylint: disable=invalid-name
+			return int(string, 16)
+
+		@staticmethod
+		def FIXED_SIZE_INTEGER(string):  # pylint: disable=invalid-name
+			return FixedSizeInteger(string)
+
+		@staticmethod
+		def fixed_size_buffer(tokens):  # pylint: disable=invalid-name
+			return FixedSizeBuffer(tokens[0])
+
+		# endregion
+
+		# region comment
+
+		@staticmethod
+		def comment(tokens):
+			return Comment(tokens[0])
+
+		@staticmethod
+		def statement(tokens):
+			tokens[1].comment = tokens[0]
+			return tokens[1]
+
+		@staticmethod
+		def _remove_comments(tokens):
+			return [token for token in tokens if not isinstance(token, Comment)]
+
+		# endregion
+
+		# region attribute
+
+		@staticmethod
+		def enum_attribute(tokens):
+			return Attribute(tokens)
+
+		@staticmethod
+		def enum_attributes(tokens):
+			return tokens  # forward array upstream
+
+		@staticmethod
+		def field_attribute(tokens):
+			return Attribute(tokens)
+
+		@staticmethod
+		def field_attributes(tokens):
+			return tokens  # forward array upstream
+
+		@staticmethod
+		def struct_attribute(tokens):
+			return Attribute(tokens)
+
+		@staticmethod
+		def struct_attributes(tokens):
+			return tokens  # forward array upstream
+
+		# endregion
+
+		# region alias
+
+		@staticmethod
+		def alias(tokens):
+			return Alias(tokens)
+
+		# endregion
+
+		# region enum
+
+		@staticmethod
+		def enum(tokens):
+			enum_model = Enum(CatbufferTransformer._remove_comments(tokens[1:]))
+			enum_model.attributes = tokens[0]
+			return enum_model
+
+		@staticmethod
+		def enum_child(tokens):
+			return CatbufferTransformer.statement(tokens)
+
+		@staticmethod
+		def enum_value(tokens):
+			return EnumValue(tokens)
+
+		# endregion
+
+		# region struct
+
+		@staticmethod
+		def struct(tokens):
+			struct_model = Struct(CatbufferTransformer._remove_comments(tokens[1:]))
+			struct_model.attributes = tokens[0]
+			return struct_model
+
+		@staticmethod
+		def struct_child(tokens):
+			return CatbufferTransformer.statement(tokens)
+
+		@staticmethod
+		def struct_inline(tokens):
+			return StructInlinePlaceholder(tokens)
+
+		@staticmethod
+		def struct_field(tokens):
+			field_model = StructField(tokens[1:])
+			field_model.attributes = tokens[0]
+			return field_model
+
+		@staticmethod
+		def struct_field_const(tokens):
+			return StructField(tokens, 'const')
+
+		@staticmethod
+		def struct_field_reserved(tokens):
+			return StructField(tokens, 'reserved')
+
+		@staticmethod
+		def struct_field_sizeof(tokens):
+			return StructField(tokens, 'sizeof')
+
+		@staticmethod
+		def struct_field_inline(tokens):
+			return StructField(tokens, 'inline')
+
+		@staticmethod
+		def conditional_expression(tokens):
+			return Conditional(tokens)
+
+		# endregion
+
+		# region array
+
+		@staticmethod
+		def array_expression(tokens):
+			return Array(tokens)
+
+		# endregion
+
+	return Lark.open(
+		'grammar/catbuffer.lark',
+		rel_to=__file__,
+		parser='lalr',
+		postlex=CatbufferIndenter(),
+		transformer=CatbufferTransformer())

catparser/DisplayType.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class DisplayType(Enum):
+	"""Enumeration of AST object display types."""
+
+	UNSET = 0
+	INTEGER = 1
+	BYTE_ARRAY = 2
+	TYPED_ARRAY = 3
+	ENUM = 4
+	STRUCT = 5
+
+	@property
+	def is_array(self):
+		"""Returns true if this display type is an array type."""
+		return self in (DisplayType.BYTE_ARRAY, DisplayType.TYPED_ARRAY)

catparser/__main__.py

@@ -0,0 +1,138 @@
+import argparse
+import importlib
+import sys
+from pathlib import Path
+
+import yaml
+from lark import Tree
+
+from .ast import AstException, Statement
+from .AstPostProcessor import AstPostProcessor
+from .AstValidator import AstValidator
+from .CatsLarkParser import create_cats_lark_parser
+
+
+def print_error(message):
+	print(f'\033[31m{message}\033[39m')
+
+
+class LarkMultiFileParser:
+	"""Multifile CATS parser implementation using Lark."""
+
+	def __init__(self):
+		self.parser = create_cats_lark_parser()
+		self.dirname = None
+
+		self.type_descriptors = []
+		self.processed_filepaths = []
+
+	def set_include_path(self, include_path):
+		self.dirname = Path(include_path)
+
+	def parse(self, filepath):
+		if filepath in self.processed_filepaths:
+			return []
+
+		print(f'processing \033[33m{filepath}\033[39m...')
+		self.processed_filepaths.append(filepath)
+
+		with open(filepath, 'rt', encoding='utf8') as infile:
+			contents = infile.read()
+
+		parse_result = self.parser.parse(contents)
+		if isinstance(parse_result, Statement):
+			return [parse_result]
+
+		descriptors = []
+		unprocessed_trees = [child for child in parse_result.children if isinstance(child, Tree)]
+		for unprocessed_tree in unprocessed_trees:
+			if 'import' == unprocessed_tree.data:
+				descriptors += self.parse(self.dirname / unprocessed_tree.children[0])
+			else:
+				raise AstException(f'found unexpected unprocessed tree "{unprocessed_tree.data}"')
+
+		# filter out trees and unattached comments
+		return descriptors + [descriptor for descriptor in parse_result.children if isinstance(descriptor, Statement)]
+
+
+def _validate(raw_type_descriptors, stage, mode):
+	validator = AstValidator(raw_type_descriptors)
+	validator.set_validation_mode(mode)
+	validator.validate()
+	if validator.errors:
+		print_error(f'[ERRORS DETECTED AT STAGE {stage}]')
+		for error in validator.errors:
+			print(f' + {error}')
+
+		sys.exit(2)
+
+
+def _load_generator_class(generator_full_name):
+	generator_class_name = Path(generator_full_name).suffix[1:]
+
+	print()
+	print(f'loading generator {generator_class_name} from {generator_full_name}')
+
+	generator_module = importlib.import_module(generator_full_name)
+	generator_class = getattr(generator_module, generator_class_name)
+
+	print(f'loaded generator: {generator_class}')
+	print()
+
+	return generator_class
+
+
+class NoAliasDumper(yaml.SafeDumper):
+	def ignore_aliases(self, data):
+		return True
+
+
+def main():
+	parser = argparse.ArgumentParser(
+		prog=None if globals().get('__spec__') is None else f'python -m {__spec__.name.partition(".")[0]}',
+		description='CATS code generator'
+	)
+	parser.add_argument('-s', '--schema', help='input CATS file', required=True)
+	parser.add_argument('-i', '--include', help='schema root directory', required=True)
+	parser.add_argument('-o', '--output', help='yaml output file')
+	parser.add_argument('-g', '--generator', help='generator class to use to produce output files (defaults to YAML output)')
+	parser.add_argument('-q', '--quiet', help='do not print type descriptors to console', action='store_true')
+	args = parser.parse_args()
+
+	file_parser = LarkMultiFileParser()
+	file_parser.set_include_path(args.include)
+
+	try:
+		raw_type_descriptors = file_parser.parse(args.schema)
+	except (AstException, OSError) as ex:
+		print_error(str(ex))
+		sys.exit(1)
+
+	processor = AstPostProcessor(raw_type_descriptors)
+
+	_validate(raw_type_descriptors, 'PRE EXPANSION', AstValidator.Mode.PRE_EXPANSION)
+
+	processor.apply_attributes()
+	processor.expand_named_inlines()
+	processor.expand_unnamed_inlines()
+
+	_validate(raw_type_descriptors, 'POST EXPANSION', AstValidator.Mode.POST_EXPANSION)
+
+	type_descriptors = [model.to_legacy_descriptor() for model in processor.type_descriptors]
+
+	if not args.quiet:
+		# dump parsed type descriptors to console
+		yaml.dump(type_descriptors, sys.stdout, Dumper=NoAliasDumper)
+
+	if args.output:
+		if args.generator:
+			generator_class = _load_generator_class(args.generator)
+			generator_class.generate(processor.type_descriptors, args.output)
+		else:
+			# save YAML to file
+			with open(args.output, 'wt', encoding='utf8') as out:
+				yaml.dump(type_descriptors, out, Dumper=NoAliasDumper)
+
+
+if '__main__' == __name__:
+	main()