PKJGףalteraparser/__init__.pyfrom alteraparser.syntaxgraph.matcher_vertex import MatcherVertex from alteraparser.syntaxgraph.final_vertex import FinalVertex from alteraparser.syntaxgraph.vertex_group import Multiples, Branches, VertexGroup def group(name=None, is_unique=False, transform_ast_fn=None): def init(self): VertexGroup.__init__(self) if name: self._VertexGroup__start.name = name self._VertexGroup__end.name = name self._VertexGroup__end.is_rule_end = is_unique if transform_ast_fn: self._VertexGroup__end.transform_ast_fn = transform_ast_fn def factory(on_expand_fn): return type(on_expand_fn.__name__, (VertexGroup,), {'__init__': init, '_on_expand': on_expand_fn}) return factory def optional(element): return Multiples(element, max_occur=1) def many(element): return Multiples(element) def one_to_many(element): return Multiples(element, min_occur=1) def fork(*branches): res = Branches() for branch in branches: if isinstance(branch, list): res.add_branch(branch) else: res.add_branch([branch]) return res def seq(sep, *elements): new_elements = [] for el in elements: if new_elements: new_elements.append(sep) new_elements.append(el) return fork(new_elements) def grammar(name, branches, transform_ast_fn=None): res = fork(*branches).set_name(name) if transform_ast_fn: res = res.transform_ast(transform_ast_fn) res.connect(FinalVertex()) return res def single_char(ch): return MatcherVertex([ch]) def char_range(ch_from, ch_to): return MatcherVertex([chr(i) for i in range(ord(ch_from), ord(ch_to) + 1)]) def characters(*chars): return MatcherVertex(chars) def keyword(kw, case_sensitive=True, name='key'): branch = [] if case_sensitive: for ch in kw: branch.append(single_char(ch)) else: for ch in kw: elem = fork([single_char(ch.lower())], [single_char(ch.upper())]) branch.append(elem) return fork(branch).set_name(name) def token(element, name='token'): return fork(element).set_name(name) PKꆆG]JS S alteraparser/parser.pyfrom alteraparser.ast import AST, TextNode from alteraparser.io.string_input import StringInput from alteraparser.syntaxgraph.match_finder import MatchFinder class ParseError(RuntimeError): pass class Parser(object): def __init__(self, grammar): self.__grammar = grammar self.__debug = False def debug_mode(self, debug=True): self.__debug = debug return self def parse(self, input_stream): finder = MatchFinder(input_stream).debug_mode(self.__debug) self.__grammar.get_dock_vertex().walk(finder) if not finder.stopped: return self.__create_ast(finder.path) else: raise ParseError(self.__get_unparsed_text(finder.path)) def parse_string(self, code_str): return self.parse(StringInput(code_str)) def parse_file(self, file_path): f = open(file_path, "r") code_lines = f.readlines() f.close() code = ''.join(code_lines) return self.parse_string(code) def __create_ast(self, path): root = None stack = [] text = '' for vertex, ch in path: if vertex.is_group_start(): if text and stack: parent = stack[-1] parent.add_child(TextNode(text)) text = '' node = AST(vertex.name, vertex.id) stack.append(node) if self.__debug: print('PUSH -> {}'.format(vertex.name)) print(self.__stack_to_string(stack)) elif vertex.is_group_end(): node = stack.pop() if self.__debug: print('POP <- {}'.format(vertex.name)) print(self.__stack_to_string(stack)) node.add_child(TextNode(text)) text = '' id_ = node.id transformed_node = vertex.transform_ast_fn(node) # ID must not be changed! transformed_node.id = id_ if stack: parent = stack[-1] if not vertex.ignore: parent.add_child(transformed_node) else: root = transformed_node if ch is not None: text += ch return root @staticmethod def __get_unparsed_text(path): return ''.join([ch for _, ch in path if ch is not None]) @staticmethod def __stack_to_string(stack): res = [node.name for node in stack if node.name] res = '[' + ','.join(res) + ']' return res PKV7uG1e alteraparser/ast.pyimport os class AST(object): def __init__(self, name, id='', text=''): self.__name = name self.id = id self.__children = [] if text: self.__children.append(TextNode(text)) def get_name(self): return self.__name name = property(get_name) def get_text(self): return ''.join([child.text for child in self.__children]) text = property(get_text) def get_own_text(self): return ''.join([tn.text for tn in self.__children if isinstance(tn, TextNode)]) own_text = property(get_own_text) def add_child(self, child, pos=None): if pos is None: self.__children.append(child) else: self.__children.insert(pos, child) def get_children(self): return self.__children children = property(get_children) def get_ast_children(self): return [ast for ast in self.__children if isinstance(ast, AST)] ast_children = property(get_ast_children) def get_children_by_name(self, name, recursive=True): res = [] for child in self.__children: if not isinstance(child, AST): continue if child.__name == name: res.append(child) elif recursive and not child.__name: res += child.get_children_by_name(name) return res def get_children_by_id(self, id_, recursive=True): res = [] for child in self.__children: if not isinstance(child, AST): continue if child.id == id_: res.append(child) elif recursive and not child.__name: res += child.get_children_by_id(id_) return res def __getitem__(self, key): if key[0] == '#': return self.get_children_by_id(key[1:]) else: return self.get_children_by_name(key) def to_xml(self, indent_size=2): intro = '' return intro + os.linesep + self._to_xml('', 0, indent_size) def _to_xml(self, xml, indent, indent_size): name = self.__name or 'ast' new_xml = ' ' * (indent*indent_size) new_xml += '<{}'.format(name) if self.id: new_xml += ' id = "{}"'.format(self.id) text = self.own_text children = self.ast_children closed = False if text or children: new_xml += '>' else: new_xml += '/>' closed = True if text: new_xml += text if children: for child in children: new_xml = child._to_xml(new_xml, indent+1, indent_size) if not closed: if children: new_xml += os.linesep + ' ' * (indent*indent_size) new_xml += ''.format(name) if xml: return xml + os.linesep + new_xml else: return new_xml class TextNode(object): def __init__(self, text): self.text = text PKZpGalteraparser/bnf/__init__.pyPKSG Q{${$alteraparser/bnf/grammar.pyfrom alteraparser import * from alteraparser.ast import AST # Define BNF-like grammar quote = single_char("'") non_quote = quote.clone().negate() question_mark = single_char('?') plus = single_char('+') star = single_char('*') caret = single_char('^') ampersand = token(single_char('&'), 'ampersand')\ .transform_ast(lambda ast: AST('no-ws')) opt_ws = keyword('&?').transform_ast(lambda ast: AST('optional-ws')) par_open = single_char('(') par_close = single_char(')') bracket_open = single_char('[') bracket_close = single_char(']') pipe = token(single_char('|'), 'pipe') assign = single_char('=') dot = single_char('.') semicolon = token(single_char(';'), 'semicolon') hash_char = single_char('#') alpha = char_range('a', 'z') num = char_range('0', '9') alpha_num = fork(alpha, num) underscore = single_char('_') whitespace = token(characters(' ', '\t', '\n'), 'ws').set_ignore() newline = keyword('', name='newline') tab = keyword('', name='tab') space = keyword('', name='space') ws = keyword('WHITESPACE', name='WHITESPACE') def ws_trnsf(ast): res = AST('WHITESPACE') for child in ast.ast_children: res.add_child(child) return res ws = ws.transform_ast(ws_trnsf) def cardinality_trnsf(ast): children = ast.children if len(children) == 1: no_whitespace = None value = children[0].text else: opt_no_ws = children[0] if opt_no_ws.ast_children: no_whitespace = opt_no_ws.ast_children[0] else: no_whitespace = None value = children[1].text res = AST('cardinality') element = { '?': 'zero-to-one', '+': 'one-to-many', '*': 'many' }[value] res.add_child(AST(element)) if no_whitespace: res.add_child(no_whitespace) return res cardinality = fork( question_mark, [optional(ampersand), plus], [optional(ampersand), star])\ .set_name('cardinality')\ .transform_ast(cardinality_trnsf) def special_char_trnsf(ast): name = ast.children[0].name return AST(name) special_char = fork(newline, tab, space).transform_ast(special_char_trnsf) def esc_trnsf(ast): return AST('esc', text="'") esc = fork([single_char('\\'), quote])\ .set_name('esc')\ .transform_ast(esc_trnsf) def terminal_trnsf(ast): return AST('term', text=ast.text[1:-1]) terminal = fork([quote, many(fork(esc, non_quote)), quote]).set_name('term')\ .transform_ast(terminal_trnsf) def rule_name_trnsf(ast): return AST('rule-name', text=ast.text) rule_name = fork([alpha, many(fork(alpha_num, fork([underscore, alpha_num])))])\ .set_name('rule_name')\ .set_unique()\ .transform_ast(rule_name_trnsf) def id_trnsf(ast): return AST('id', text=ast.text) id_name = fork([alpha, many(fork(alpha_num, fork([underscore, alpha_num])))])\ .set_name('id_name')\ .set_unique()\ .transform_ast(id_trnsf) def prod_rule_trnsf(ast): is_grammar = False is_unique = False annotations = ast['annotations'] if annotations: if annotations[0]['grammar']: is_grammar = True if annotations[0]['unique']: is_unique = True if not is_grammar: res = AST('rule') else: res = AST('grammar') r_name = ast['rule-name'] if r_name: r_name = r_name[0].text else: r_name = 'WHITESPACE' res.add_child(AST('name', text=r_name)) rhs = ast['#rhs'][0] rhs.id = '' res.add_child(rhs) res.add_child(AST('unique', text=str(is_unique).lower())) return res @group(name='rule', is_unique=True, transform_ast_fn=prod_rule_trnsf) def prod_rule_stmt(self, start, end): global whitespace, rule_name, ws, assign, semicolon start > many(whitespace) > \ many(annotation_stmt().set_id('annot')) > \ fork(rule_name, ws) > \ many(whitespace) > \ assign.clone() > \ many(whitespace) > \ expr_stmt().set_id('rhs') > \ many(whitespace) > \ semicolon.clone() > \ end def annotation_trnsf(ast): res = AST('annotations') for node in ast['#annot']: name = node.name if name == 'grammar': res.add_child(AST('grammar')) elif name == 'unique': res.add_child(AST('unique')) return res @group(name='annotation', is_unique=True, transform_ast_fn=annotation_trnsf) def annotation_stmt(self, start, end): wspace = characters(' ', '\t') nl = single_char('\n') v = start.clone() annotations = fork(keyword('@grammar', name='grammar').set_id('annot'), keyword('@unique', name='unique').set_id('annot')) start > many(fork(wspace, nl)) > annotations > \ many(wspace) > nl.clone() > many(wspace) > v v > start v > end def expr_transf(ast): branches = ast['#branch'] if len(branches) == 1: res = branches[0] else: res = AST('branches') for branch in branches: branch.id = '' res.add_child(branch) return res @group(transform_ast_fn=expr_transf) def expr_stmt(self, start, end): global pipe, whitespace start > branch_stmt().set_id('branch') > \ many(fork([ one_to_many(whitespace), pipe, one_to_many(whitespace), branch_stmt().set_id('branch')])) > end def branch_trnsf(ast): res = AST('branch') for content in ast['#content']: if content.ast_children: node_fork, opt_id, opt_cardinal = content.ast_children node = node_fork.ast_children[0] if opt_id.ast_children: id_node = opt_id.ast_children[0].ast_children[0] node.add_child(AST('id', text=id_node.text)) if opt_cardinal.ast_children: cardinal = opt_cardinal.ast_children[0] node.add_child(cardinal) else: node = content node.id = '' res.add_child(node) return res @group('branch', transform_ast_fn=branch_trnsf) def branch_stmt(self, start, end): global ws, terminal, rule_name, whitespace,\ special_char, cardinality, ampersand, hash_char, opt_ws v = start.clone() start > fork([fork( ws, terminal, rule_name, range_stmt(), charset_stmt(), special_char, comp_stmt()), optional(fork([hash_char, id_name])), optional(cardinality)]).set_id('content') >\ v v > one_to_many(whitespace) >\ optional(fork([ampersand.set_id('content'), one_to_many(whitespace)])) > start v > one_to_many(whitespace) >\ optional(fork([opt_ws.set_id('content'), one_to_many(whitespace)])) > start v.connect(end) def comp_trnsf(ast): res = AST('comp') expr = ast['#expr'][0] expr.id = '' res.add_child(expr) return res @group('comp', is_unique=True, transform_ast_fn=comp_trnsf) def comp_stmt(self, start, end): global par_open, par_close, whitespace start > par_open.clone() > \ optional(whitespace) > \ expr_stmt().set_id('expr') > \ optional(whitespace) > \ par_close.clone() > \ end def range_trnsf(ast): res = AST('range') from_ = ast['#from'][0] to = ast['#to'][0] res.add_child(AST('from', text=from_.text)) res.add_child(AST('to', text=to.text)) return res @group(name='range', is_unique=True, transform_ast_fn=range_trnsf) def range_stmt(self, start, end): global dot, terminal from_ = terminal.set_id('from') to = terminal.set_id('to') start > from_ > dot.clone() > dot.clone() > to > end def charset_trnsf(ast): res = AST('charset') fork_node = ast.children[0] opt_neg = fork_node['#neg'] if opt_neg[0].text: res.add_child(AST('negate')) char_elements = fork_node['#char-element'] for char_elem in char_elements: char_elem.id = '' if char_elem.text: res.add_child(AST('char', text=char_elem.text)) else: # special character special_node = char_elem.ast_children[0] special_node.id = '' res.add_child(special_node) return res @group(name='charset', is_unique=True, transform_ast_fn=charset_trnsf) def charset_stmt(self, start, end): global bracket_open, bracket_close, \ caret, special_char no_bracket_close = bracket_close.clone().negate() start > fork([ bracket_open, optional(caret).set_id('neg'), one_to_many(fork( special_char.set_id('special'), no_bracket_close, ).set_id('char-element')), bracket_close ]) > end def bnf_grammar_trnsf(ast): res = AST('bnf-grammar') for child in ast['#grammar-element']: child.id = '' res.add_child(child) return res bnf_grammar = grammar('bnf', [one_to_many( fork([many(whitespace), prod_rule_stmt().set_id('grammar-element'), many(whitespace)]))], bnf_grammar_trnsf) PK tG2& & (alteraparser/syntaxgraph/match_finder.pyfrom .processor import Processor, ProcessingResult from .vertex import VertexCategory class MatchFinder(Processor): def __init__(self, input): self.__path = [] self.__input = input self.__buffer = [] self.__match_char = None self.__stopped = False self.__debug = False def process(self, vertex, path): if self.__stopped: return ProcessingResult.STOP if not self.__match_char: self.__match_char = self.__get_next_char() if self.__match_char: return self.__process_with_char_search(vertex) else: return self.__process_without_char_search(vertex) def undo(self, vertex, path): if not self.__path: return v, ch = self.__path[-1] if vertex is v: self.__path.pop() if ch is not None: if self.__match_char: self.__buffer.append(self.__match_char) self.__match_char = None self.__buffer.append(ch) if v.is_group_end() and v.is_rule_end: self.__stopped = True def get_path(self): return self.__path path = property(get_path) def get_stopped(self): return self.__stopped stopped = property(get_stopped) def debug_mode(self, debug=True): self.__debug = debug return self def __process_with_char_search(self, vertex): catg = vertex.get_category() if catg == VertexCategory.MATCHER: if self.__debug: print("Searching for '{}'".format(self.__match_char)) print(vertex) if vertex.matches(self.__match_char): self.__path.append((vertex, self.__match_char)) if self.__debug: print('Match: {}'.format(self.__match_char)) self.__match_char = None return ProcessingResult.CONTINUE else: return ProcessingResult.GO_BACK elif catg == VertexCategory.FINAL: return ProcessingResult.GO_BACK else: self.__path.append((vertex, None)) return ProcessingResult.CONTINUE def __process_without_char_search(self, vertex): catg = vertex.get_category() if catg == VertexCategory.MATCHER: return ProcessingResult.GO_BACK elif catg == VertexCategory.FINAL: return ProcessingResult.STOP else: self.__path.append((vertex, None)) return ProcessingResult.CONTINUE def __get_next_char(self): if self.__buffer: return self.__buffer.pop() else: if self.__input.has_next_char(): return self.__input.get_next_char() else: return NonePK˅iGeL$$$alteraparser/syntaxgraph/dockable.pyclass Dockable(object): def connect(self, dockable): raise NotImplemented def __gt__(self, other): self.connect(other) return other def get_dock_vertex(self): """ Return vertex that can be docked """ raise NotImplemented PK qgG$alteraparser/syntaxgraph/clonable.pyclass Clonable(object): def clone(self): cloned_obj = self.__class__() cloned_obj._on_clone_creation(self) return cloned_obj def _on_clone_creation(self, original): raise NotImplemented PKԝiGށ"alteraparser/syntaxgraph/vertex.pyfrom .abstract_vertex import AbstractVertex from .dockable import Dockable class VertexCategory: NORMAL = 1 GROUP_START = 2 GROUP_END = 3 MATCHER = 4 FINAL = 5 class Vertex(AbstractVertex, Dockable): def __init__(self, category = VertexCategory.NORMAL): AbstractVertex.__init__(self) self.__successors = [] self._category = category def get_category(self): return self._category def is_normal(self): return self._category == VertexCategory.NORMAL def is_group_start(self): return self._category == VertexCategory.GROUP_START def is_group_end(self): return self._category == VertexCategory.GROUP_END def is_matcher(self): return self._category == VertexCategory.MATCHER def is_final(self): return self._category == VertexCategory.FINAL def num_successors(self): return len(self.__successors) def nth_successor(self, idx): if 0 <= idx < len(self.__successors): return self.__successors[idx] else: return None def connect(self, dockable): self.__successors.append(dockable.get_dock_vertex()) return dockable def get_dock_vertex(self): return self def _on_clone_creation(self, original): for successor in original.__successors: self.__successors.append(successor.clone()) PK0iGLӔ(alteraparser/syntaxgraph/final_vertex.pyfrom .vertex import Vertex, VertexCategory class FinalVertex(Vertex): def __init__(self): Vertex.__init__(self, VertexCategory.FINAL)PKbGM%alteraparser/syntaxgraph/processor.py class Processor(object): def process(self, vertex, path): pass def undo(self, vertex, path): pass class ProcessingResult: CONTINUE = 0 GO_BACK = 1 STOP = 2PKG2&&(alteraparser/syntaxgraph/vertex_group.pyfrom .dockable import Dockable from .clonable import Clonable from .vertex import Vertex, VertexCategory class StartVertex(Vertex): def __init__(self, vertex_group): Vertex.__init__(self, VertexCategory.GROUP_START) self.__group = vertex_group self.name = None self.id = None def num_successors(self): if not self.__group._VertexGroup__expanded: self.__group.expand() return Vertex.num_successors(self) def nth_successor(self, idx): if not self.__group._VertexGroup__expanded: self.__group.expand() return Vertex.nth_successor(self, idx) class EndVertex(Vertex): def __init__(self): Vertex.__init__(self, VertexCategory.GROUP_END) self.name = None self.id = None self.ignore = False self.transform_ast_fn = lambda ast: ast self.is_rule_end = False class VertexGroup(Dockable, Clonable): def __init__(self): self.__expanded = False self.__start = StartVertex(self) self.__end = EndVertex() def set_name(self, name): res = self.clone() res.__start.name = name res.__end.name = name return res def get_name(self): return self.__start.name name = property(get_name, set_name) def set_id(self, id): res = self.clone() res.__start.id = id res.__end.id = id return res def get_id(self): return self.__start.id id = property(get_id, set_id) def set_ignore(self, ignore=True): self.__end.ignore = ignore return self def transform_ast(self, transformer_fn): res = self.clone() res._VertexGroup__end.transform_ast_fn = transformer_fn return res def set_unique(self, is_unique=True): res = self.clone() res._VertexGroup__end.is_rule_end = is_unique return res def connect(self, dockable): self.__end.connect(dockable) return dockable def get_dock_vertex(self): return self.__start def expand(self): if not self.__expanded: start = Vertex() end = Vertex() self.__start.connect(start) end.connect(self.__end) self._on_expand(start, end) self.__expanded = True def _on_expand(self, start, end): pass def _on_clone_creation(self, original): self.__start.name = self.__end.name = original.__start.name self.__start.id = self.__end.id = original.__start.id self.__end.ignore = original.__end.ignore self.__end.transform_ast_fn = original.__end.transform_ast_fn self.__end.is_rule_end = original.__end.is_rule_end class Multiples(VertexGroup): def __init__(self, element=None, min_occur=0, max_occur=None): VertexGroup.__init__(self) if element: self.__element = element.clone() else: self.__element = None self.__min_occur = min_occur self.__max_occur = max_occur def _on_expand(self, start, end): current = start for _ in range(self.__min_occur): current = current.connect(self.__element.clone()) min_end = current if self.__max_occur is None: if current is not start: current.connect(current) else: elem = self.__element.clone() start.connect(elem).connect(start) elem.connect(end) else: delta = self.__max_occur - self.__min_occur for _ in range(delta): current = current.connect(self.__element.clone()) current.connect(end) min_end.connect(end) def _on_clone_creation(self, original): VertexGroup._on_clone_creation(self, original) self.__element = original.__element.clone() self.__min_occur = original.__min_occur self.__max_occur = original.__max_occur class Branches(VertexGroup): def __init__(self): VertexGroup.__init__(self) self.__branches = [] def add_branch(self, elements): self.__branches.append([el.clone() for el in elements]) def _on_expand(self, start, end): for branch in self.__branches: curr = start for elem in branch: curr = curr.connect(elem) curr.connect(end) def _on_clone_creation(self, original): VertexGroup._on_clone_creation(self, original) self.__branches = [[el.clone() for el in orig_branch] for orig_branch in original.__branches] PKsG)n6*alteraparser/syntaxgraph/matcher_vertex.pyfrom .vertex import Vertex, VertexCategory class MatcherVertex(Vertex): def __init__(self, chars=[], negated=False): Vertex.__init__(self, VertexCategory.MATCHER) self.__chars = set(chars) self.__negated = negated def matches(self, ch): if not self.__negated: return ch in self.__chars else: return ch not in self.__chars def negate(self, neg=True): self.__negated = neg return self def add(self, ch): self.__chars.append(ch) return self def _on_clone_creation(self, original): Vertex._on_clone_creation(self, original) self.__chars = set(original.__chars) self.__negated = original.__negated def __str__(self): chars = '[' + ','.join(self.__chars) + ']' if self.__negated: return 'MATCH: NOT {}'.format(chars) else: return 'MATCH: {}'.format(chars) PKiGL$alteraparser/syntaxgraph/__init__.py__author__ = 'bolle' PKmphG+alteraparser/syntaxgraph/abstract_vertex.pyfrom .processor import ProcessingResult from .clonable import Clonable class AbstractVertex(Clonable): def __init__(self): Clonable.__init__(self) def num_successors(self): raise NotImplemented def nth_successor(self, idx): raise NotImplemented def walk(self, processor): path = [] current = self result = None while True: result = processor.process(current, path) if result in [ProcessingResult.CONTINUE, None]: next = self.__continue(current, path) if not next: next = self.__back(path, processor) elif result == ProcessingResult.GO_BACK: next = self.__back(path, processor) else: break if next is None: break current = next return result @staticmethod def __continue(current, path): if current.num_successors() > 0: next = current.nth_successor(0) path.append((current, 0)) else: next = None return next @staticmethod def __back(path, processor): next_vertex = None while path: vertex, idx = path.pop() if idx < vertex.num_successors() - 1: next_vertex = vertex.nth_successor(idx + 1) path.append((vertex, idx + 1)) return next_vertex else: processor.undo(vertex, path) return next_vertex PK'NhG6Balteraparser/io/string_input.pyfrom .input_stream import InputStream class StringInput(InputStream): def __init__(self, string_data): self.__string_data = string_data self.__cursor = -1 def get_next_char(self): self.__cursor += 1 if self.__cursor < len(self.__string_data): return self.__string_data[self.__cursor] else: return None def has_next_char(self): return self.__cursor + 1 < len(self.__string_data)PKU{G1dalteraparser/io/file_input.pyfrom .input_stream import InputStream class FileInput(InputStream): def __init__(self, file_path): f = open(file_path, 'r') self.__lines = f.readlines() f.close() self.__line_num = 0 self.__col_num = 0 def get_next_char(self): line = self.__lines[self.__line_num] next_char = line[self.__col_num] self.__col_num += 1 if self.__col_num >= len(line): self.__line_num += 1 self.__col_num = 0 return next_char def has_next_char(self): if self.__line_num >= len(self.__lines): return False else: line = self.__lines[self.__line_num] return self.__col_num <= len(line) - 1 PKG$$alteraparser/io/output.pyimport os class Output(object): def write(self, text): pass def writeln(self, text=''): self.write(text + os.linesep) class ConsoleOutput(Output): def write(self, text): print(text, end='') class BufferedOutput(Output): def __init__(self): self.__line = '' self.__lines = [] def write(self, text): self.__line += text def writeln(self, text=''): self.__line += text self.__lines.append(self.__line) self.__line = '' def get_lines(self): if self.__line: return self.__lines + [self.__line] else: return self.__lines class FileOutput(Output): def __init__(self, file_path): self.__file_path = file_path self.__file = None def open(self): if self.__file: self.__file.close() self.__file = open(self.__file_path, 'w') def close(self): self.__file.close() self.__file = None def write(self, text): self.__file.write(text)PKDhGMalteraparser/io/input_stream.pyclass InputStream(object): def has_next_char(self): raise NotImplemented def get_next_char(self): raise NotImplementedPK{DhGalteraparser/io/__init__.pyPK퐆G>~JF+F+!alteraparser/codegen/generator.pyfrom alteraparser.parser import Parser from alteraparser.bnf.grammar import bnf_grammar from alteraparser.io.output import BufferedOutput import re class Generator(object): def __init__(self, output): self.__bnf_parser = Parser(bnf_grammar) self.__output = output self.__indent_level = 0 self.__indent_size = 4 self.__fn_id_creator = FnIdCreator() self.__functions = {} self.__edit_section_start = re.compile(r'\s*#--beginedit\s+(\S+)\s*$') self.__edit_section_end = re.compile(r'\s*#--endedit\s*$') def generate_parser(self, grammar_input_stream, edit_sections={}): self.__writeln('from alteraparser import *') self.__writeln('from alteraparser.ast import AST') self.__writeln('from alteraparser.parser import Parser') self.__writeln() self.__writeln() ast = self.__bnf_parser.parse(grammar_input_stream) grammar_name = self.__find_grammar_name(ast) self.__writeln("def create_{}_parser():".format(grammar_name)) self.__indent() self.__writeln("return Parser({}())".format(grammar_name)) self.__dedent() self.__writeln() self.__writeln() for rule in ast.ast_children: if rule.name != 'grammar': self.__generate_rule(rule, edit_sections) else: self.__generate_grammar(rule, edit_sections) self.__generate_internal_functions() def __find_grammar_name(self, ast): for rule in ast.ast_children: if rule.name == 'grammar': return rule.ast_children[0].text return '' def __generate_grammar(self, grammar, edit_sections): name = grammar.ast_children[0].text.lower() self.__writeln("def _{}_trnsf(ast):".format(name)) self.__indent() self.__generate_edit_section(name, edit_sections) self.__dedent() self.__writeln() self.__writeln() self.__writeln('def {}():'.format(name)) node = grammar.ast_children[1] branches = [] if node.name == 'branches': for it in node.ast_children: branches.append(it) else: branches.append(node) self.__indent() self.__writeln('branches = []') for branch in branches: call = self.__create_call(branch) self.__writeln('branches.append({})'.format(call)) line = "return grammar('{0}', branches, _{0}_trnsf)".format(name) self.__writeln(line) self.__dedent() self.__writeln() self.__writeln() def __generate_edit_section(self, section_name, edit_sections): self.__writeln('#--beginedit {}'.format(section_name)) if section_name in edit_sections: lines = edit_sections[section_name] for line in lines: self.__output.writeln(line) #<-- ignore indentation! else: self.__writeln('return ast') self.__writeln('#--endedit') def __generate_rule(self, rule, edit_sections): rule_name = rule.ast_children[0].text.lower() unique = rule.ast_children[2].text == 'true' self.__writeln("def _{}_trnsf(ast):".format(rule_name)) self.__indent() self.__generate_edit_section(rule_name, edit_sections) self.__dedent() self.__writeln() self.__writeln() line = "@group(name='{0}', is_unique={1}, transform_ast_fn=_{0}_trnsf)".format(rule_name, unique) self.__writeln(line) line = 'def _{}(self, start, end):'.format(rule_name) self.__writeln(line) self.__indent() self.__generate_fn_body(rule.ast_children[1]) self.__dedent() self.__writeln() self.__writeln() def __generate_fn_body(self, ast): body = self.__create_fn_body(ast) for line in body: self.__output.writeln(line) # indentation already done def __create_fn_body(self, ast): saved_output = self.__output buf_output = BufferedOutput() self.__output = buf_output if ast.name == 'branches': self.__generate_branches_body(ast) elif ast.name == 'branch': self.__generate_branch_body(ast) elif ast.name == 'comp': self.__generate_comp_body(ast) else: self.__output.writeln('pass') self.__output = saved_output return buf_output.get_lines() def __create_call(self, ast): name = ast.name if name in ['branches', 'branch', 'comp']: call = '{}()'.format(self.__create_fn(ast)) elif name == 'term': call = "keyword('{}')".format(ast.text) elif name == 'rule-name': call = '_{}()'.format(ast.own_text.lower()) elif name == 'WHITESPACE': call = '_whitespace()' elif name == 'range': ch_from = ast['from'][0].text ch_to = ast['to'][0].text call = "char_range('{}', '{}')".format(ch_from, ch_to) elif name == 'charset': negate = ast['negate'] if not negate: char_nodes = ast.ast_children else: char_nodes = ast.ast_children[1:] chars = '' prev_cname = '' relevant_items = ['char', 'space', 'tab', 'newline'] for char_node in char_nodes: cname = char_node.name if chars and prev_cname in relevant_items and cname in relevant_items: chars += ', ' if cname == 'char': chars += "'{}'".format(char_node.text) elif cname in ['space', 'tab', 'newline']: chars += "'{}'".format({ 'space': ' ', 'tab': '\\t', 'newline': '\\n' }[cname]) prev_cname = cname call = "characters({})".format(chars) if negate: call += '.negate()' elif name in ['space', 'tab', 'newline']: call = "single_char('{}')".format({ 'space': ' ', 'tab': '\\t', 'newline': '\\n' }[name]) else: raise GeneratorError("Unsupported grammar element '{}'".format(name)) id_nodes = ast['id'] if id_nodes: id_node = id_nodes[0] call = self.__add_id(call, id_node) card = ast['cardinality'] if card: call = self.__add_cardinality(call, card[0]) return call def __add_cardinality(self, call, card): use_whitespace = len(card.ast_children) == 1 mult = card.ast_children[0] if use_whitespace: ws = 'one_to_many(_whitespace())' if mult.name == 'zero-to-one': res = 'optional({})'.format(call) elif mult.name == 'one-to-many': res = 'fork([{0}, many(fork([{1}, {0}]))])'.format(call, ws) elif mult.name == 'many': res = 'optional(fork([{0},'.format(call) res += ' many(fork([{0}, {1}]))]))'.format(ws, call) else: raise GeneratorError() return res else: fn_name = { 'zero-to-one': 'optional', 'one-to-many': 'one_to_many', 'many': 'many' }[mult.name] return '{}({})'.format(fn_name, call) def __add_id(self, call, id_node): return "{}.set_id('{}')".format(call, id_node.text) def __generate_branches_body(self, branches): for branch in branches.ast_children: call = self.__create_call(branch) self.__writeln('start > {} > end'.format(call)) def __generate_branch_body(self, branch): self.__writeln('curr = start') prev_item = None for item in branch.ast_children: if item.name == 'no-ws': pass elif item.name == 'optional-ws': self.__writeln('curr = curr > many(_whitespace())') else: if prev_item and prev_item.name not in ['no-ws', 'optional-ws']: self.__writeln('curr = curr > one_to_many(_whitespace())') call = self.__create_call(item) self.__writeln('curr = curr > {}'.format(call)) prev_item = item self.__writeln('curr > end') def __generate_comp_body(self, comp): call = self.__create_call(comp.ast_children[0]) self.__writeln('start > {} > end'.format(call)) def __generate_internal_functions(self): fn_ids = list(self.__functions.keys()) fn_ids.sort() for fn_id in fn_ids: self.__writeln('@group()') self.__writeln('def {}(self, start, end):'.format(fn_id)) self.__indent() body = self.__functions[fn_id] for line in body: self.__output.writeln(line) self.__dedent() self.__writeln() self.__writeln() def __create_fn(self, ast): fn_id = self.__fn_id_creator.create_id(ast.name) self.__functions[fn_id] = self.__create_fn_body(ast) return fn_id def __indent(self): self.__indent_level += 1 def __dedent(self): self.__indent_level -= 1 def __writeln(self, text=''): text = self.__indent_level * self.__indent_size * ' ' + text self.__output.writeln(text) def scan_for_edit_sections(self, stream): edit_sections = {} section_name = '' lines = [] line = '' while stream.has_next_char(): ch = stream.get_next_char() if ch != '\n': line += ch else: if not section_name: match = self.__edit_section_start.match(line) if match: section_name = match.group(1) lines = [] else: match = self.__edit_section_end.match(line) if match: edit_sections[section_name] = lines section_name = '' lines = [] else: lines.append(line) line = '' return edit_sections class FnIdCreator(object): def __init__(self): self.__ids = {} def create_id(self, prefix): if prefix in self.__ids: id_ = self.__ids[prefix] + 1 else: id_ = 1 self.__ids[prefix] = id_ return '_{}_{}'.format(prefix, id_) class GeneratorError(Exception): passPK}xG alteraparser/codegen/__init__.pyPK&G*alteraparser-1.0.1.data/scripts/altparsgen#!python """ Script to generate a parser from a grammar file Syntax: python altparsgen.py [] """ import sys import os from alteraparser.io.file_input import FileInput from alteraparser.io.output import ConsoleOutput, FileOutput from alteraparser.codegen.generator import Generator if len(sys.argv) < 2: print('SYNTAX: python3 altparsgen.py []') exit(1) grammar_path = sys.argv[1] if len(sys.argv) > 2: out_file_path = sys.argv[2] output = FileOutput(out_file_path) else: out_file_path = '' output = ConsoleOutput() generator = Generator(output) if out_file_path: if os.path.exists(out_file_path): file_in = FileInput(out_file_path) edit_sections = generator.scan_for_edit_sections(file_in) else: edit_sections = {} output.open() else: edit_sections = {} generator.generate_parser(FileInput(grammar_path), edit_sections) if out_file_path: output.close() PK.G,alteraparser-1.0.1.dist-info/DESCRIPTION.rst============ Alteraparser ============ Introduction ============ Alteraparser is a library that provides functions to define a grammar that can be passed to a parser. Basic Usage =========== Code sample:: from alteraparser.parser import Parser from alteraparser import char_range, fork, many, grammar, ... ALPHA = fork(char_range('a', 'z'), char_range('A', 'Z')) NUM = char_range('0', '9') ALPHA_NUM = fork(ALPHA, NUM) ... variable = fork([ALPHA, many(ALPHA_NUM)]).set_name('var') ... my_grammar = grammar(variable, ...) my_parser = Parser(my_grammar) ast = my_parser.parse_file("my_code.txt") Changes ======= 0.5.0.a2: - added transform_ast method to enable transformation of AST nodes PK.G5f*alteraparser-1.0.1.dist-info/metadata.json{"classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Build Tools", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5"], "extensions": {"python.details": {"contacts": [{"email": "tbollmeier@web.de", "name": "Thomas Bollmeier", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}}}, "generator": "bdist_wheel (0.26.0)", "keywords": ["parser", "development"], "license": "MIT", "metadata_version": "2.0", "name": "alteraparser", "summary": "A simple parser and grammar definition library", "version": "1.0.1"}PK.GK, *alteraparser-1.0.1.dist-info/top_level.txtalteraparser PK.G}\\"alteraparser-1.0.1.dist-info/WHEELWheel-Version: 1.0 Generator: bdist_wheel (0.26.0) Root-Is-Purelib: true Tag: py3-none-any PK.G$%alteraparser-1.0.1.dist-info/METADATAMetadata-Version: 2.0 Name: alteraparser Version: 1.0.1 Summary: A simple parser and grammar definition library Home-page: UNKNOWN Author: Thomas Bollmeier Author-email: tbollmeier@web.de License: MIT Keywords: parser development Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Build Tools Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 ============ Alteraparser ============ Introduction ============ Alteraparser is a library that provides functions to define a grammar that can be passed to a parser. Basic Usage =========== Code sample:: from alteraparser.parser import Parser from alteraparser import char_range, fork, many, grammar, ... ALPHA = fork(char_range('a', 'z'), char_range('A', 'Z')) NUM = char_range('0', '9') ALPHA_NUM = fork(ALPHA, NUM) ... variable = fork([ALPHA, many(ALPHA_NUM)]).set_name('var') ... my_grammar = grammar(variable, ...) my_parser = Parser(my_grammar) ast = my_parser.parse_file("my_code.txt") Changes ======= 0.5.0.a2: - added transform_ast method to enable transformation of AST nodes PK.G>"  #alteraparser-1.0.1.dist-info/RECORDalteraparser/__init__.py,sha256=uJk_V2cFY67sp2__bEX1zi86E6SBkZsWK0drXObqQQQ,2293 alteraparser/ast.py,sha256=IuMudHpk4WnQTeLwHI4R9ucF25b3nOvQv76MkaDgZAs,3060 alteraparser/parser.py,sha256=QBXHMBIFW9C58x1YSN7HUOcjFaO9vEEkvCGqarawP_8,2643 alteraparser/bnf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 alteraparser/bnf/grammar.py,sha256=qslxn_hmaro-5XO4icQU1H0aubU1Lfn3sb5cbs1oAxo,9339 alteraparser/codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 alteraparser/codegen/generator.py,sha256=JNWYWdPDxq8blMIeOjBTL2dcTiRnl4ikBNNNAhyNesY,11078 alteraparser/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 alteraparser/io/file_input.py,sha256=ePmQO78aAaiZr-iEs6meDhxZqXrJGrGCt_E-GRL6Stg,740 alteraparser/io/input_stream.py,sha256=iY768vRp7sEMluSgoWK-atH-eqVhYpBY5zYcZAWlEC0,144 alteraparser/io/output.py,sha256=O2x4qSS0KjJAdYjwikOR44cvma5DHwzOaZ1y703VwrE,1060 alteraparser/io/string_input.py,sha256=U0DDNJzUp-nO1PdL2lultcZALWxZih334719aqTdVmE,466 alteraparser/syntaxgraph/__init__.py,sha256=GDBgV8QPFPqUIynBDupcyIxij3-ckszuHbGCN9rcqGM,22 alteraparser/syntaxgraph/abstract_vertex.py,sha256=5sCpWyBzUWuXsb_d4N5Afe42we8JiN6QqM_2sfSbrJY,1553 alteraparser/syntaxgraph/clonable.py,sha256=DHDlBSIbLXmj9dqd1NdJp3lpwrGWdCQsdcqe3RoGHj0,228 alteraparser/syntaxgraph/dockable.py,sha256=fWtgScydrZu3nzldhNNKXk--9sp_TU94pw1Xl5nCxWk,292 alteraparser/syntaxgraph/final_vertex.py,sha256=iN1DRCgxec6gftcuF8oyRDBOtitodq0CIYG1zZ-XYUU,148 alteraparser/syntaxgraph/match_finder.py,sha256=vCF6kp9S8h3IjZnlgE05mVSR2utozN9nQu-rRJ3bFWA,2854 alteraparser/syntaxgraph/matcher_vertex.py,sha256=37GT207ehIM2345dckL_T5IyHC3KQ7OJIA6hR32E6gw,952 alteraparser/syntaxgraph/processor.py,sha256=lNTNEcBimvoSjMFUiG7gLxDWB307DLORQ13f3pdqcNA,197 alteraparser/syntaxgraph/vertex.py,sha256=1IQ2jotjpjpWXOrQ_gXmzBJ63dlsbKM4W7XRW7hvoNU,1418 alteraparser/syntaxgraph/vertex_group.py,sha256=ZwToum10nFs1oQ1So7grUQo7f9LHhGnY5Fhl1orvDyQ,4646 alteraparser-1.0.1.data/scripts/altparsgen,sha256=8LNRZ0V1B_GZ5h3Dc1alGEgWsnGK0g7-Ey7g-C9hoXM,985 alteraparser-1.0.1.dist-info/DESCRIPTION.rst,sha256=A-u3TeG25WsfsN6aQYvBbLhQ678SpeVpDmjfk8Q6sOA,747 alteraparser-1.0.1.dist-info/METADATA,sha256=1qtLio6IeChwG6lMX6CPJuJpPtCDn44zpCqc7HoUBj0,1439 alteraparser-1.0.1.dist-info/RECORD,, alteraparser-1.0.1.dist-info/WHEEL,sha256=zX7PHtH_7K-lEzyK75et0UBa3Bj8egCBMXe1M4gc6SU,92 alteraparser-1.0.1.dist-info/metadata.json,sha256=_ui2nyaUkZwHoDShZdAfDmKp9v9HvQj1zZplyyWLIFQ,787 alteraparser-1.0.1.dist-info/top_level.txt,sha256=ZIcu6lgeNhLU0YRofCIwrguQPWJW0X6IEMd_w2ZjpY8,13 PKJGףalteraparser/__init__.pyPKꆆG]JS S + alteraparser/parser.pyPKV7uG1e alteraparser/ast.pyPKZpGalteraparser/bnf/__init__.pyPKSG Q{${$ alteraparser/bnf/grammar.pyPK tG2& & (Dalteraparser/syntaxgraph/match_finder.pyPK˅iGeL$$$1Palteraparser/syntaxgraph/dockable.pyPK qgG$Qalteraparser/syntaxgraph/clonable.pyPKԝiGށ"Ralteraparser/syntaxgraph/vertex.pyPK0iGLӔ(Xalteraparser/syntaxgraph/final_vertex.pyPKbGM%aYalteraparser/syntaxgraph/processor.pyPKG2&&(iZalteraparser/syntaxgraph/vertex_group.pyPKsG)n6*lalteraparser/syntaxgraph/matcher_vertex.pyPKiGL$palteraparser/syntaxgraph/__init__.pyPKmphG+-qalteraparser/syntaxgraph/abstract_vertex.pyPK'NhG6Bwalteraparser/io/string_input.pyPKU{G1dyalteraparser/io/file_input.pyPKG$$|alteraparser/io/output.pyPKDhGMalteraparser/io/input_stream.pyPK{DhG݁alteraparser/io/__init__.pyPK퐆G>~JF+F+!alteraparser/codegen/generator.pyPK}xG alteraparser/codegen/__init__.pyPK&G*٭alteraparser-1.0.1.data/scripts/altparsgenPK.G,alteraparser-1.0.1.dist-info/DESCRIPTION.rstPK.G5f*/alteraparser-1.0.1.dist-info/metadata.jsonPK.GK, *alteraparser-1.0.1.dist-info/top_level.txtPK.G}\\"߸alteraparser-1.0.1.dist-info/WHEELPK.G$%{alteraparser-1.0.1.dist-info/METADATAPK.G>"  #]alteraparser-1.0.1.dist-info/RECORDPK