#!/usr/bin/env python3 import json import operator import sys tokens = dict() program = './tokens.json' if len(sys.argv) > 1: program = sys.argv[1] with open(program, 'r') as json_tokens: tokens = json.loads(json_tokens.read()) def peek(index: int, tokens: list) -> dict: result = dict() if index < len(tokens): result = tokens[index] return result def find_token_indicies(key: str, tokens: list) -> list: result = list() index = 0 for token in tokens: if token["value"] == key: result.append(index) index = index + 1 return result def find_return_statement(maybe_expression, usefull_tokens) -> dict: expression = dict() for token in usefull_tokens: if maybe_expression['value'] == token['signifier'] and token['variant'] == 'function_declaration': for part in token['body']: if part['variant'] == 'return_statement': return part return expression def collect_body(index: int, tokens: list) -> list: uncollapsed_tokens = list() token = peek(index + 1, tokens) if token['variant'] != "operator": return uncollapsed_tokens maybe_expression = peek(index + 2, tokens) if maybe_expression['variant'] == 'expression': uncollapsed_tokens.append(token) return uncollapsed_tokens elif maybe_expression['variant'] == 'start_block': index = index + 2 while True: index = index + 1 collectable = peek(index, tokens) if collectable['variant'] == 'end_block': break uncollapsed_tokens.append(collectable) return uncollapsed_tokens def coaless_tokens(identifiers, line_number, usefull_tokens, variant) -> list: result = list() for identifier in identifiers: candidate_token = dict() for token in usefull_tokens: if line_number > token['line_number'] and identifier == token['signifier'] and token['variant'] == variant: candidate_token = token if candidate_token: result.append(candidate_token) return result def collapse_tokens(uncollapsed_tokens: list) -> list: usefull_tokens = list() in_block = False block = "" block_index = -1 block_lineno = -1 for index in range(0, len(uncollapsed_tokens)): token = uncollapsed_tokens[index] if token['variant'] == 'start_block': block_index = index in_block = True block_lineno = token['line_number'] if token['variant'] == 'end_block': compare_lineno = block_lineno + 1 block_lineno = token['line_number'] # Do we need to append new lines for i in range (compare_lineno, block_lineno): block = block + "\n" name = "multiline" maybe_variable = peek(block_index - 2, uncollapsed_tokens) if maybe_variable['variant'] == 'signifier': name = maybe_variable['value'] usefull_token = { "variant": "variable_declaration", "signifier": name, "type": "string", "line_number": maybe_variable['line_number'], "expression": block } usefull_tokens.append(usefull_token) # Reset for next multiline in_block = False block = str() block_index = -1 block_lineno = -1 if token['variant'] == 'syntax_error': if in_block: compare_lineno = block_lineno + 1 block_lineno = token['line_number'] # This is if we don't have a block allready if block == "": # Do we need to prepend empty lines? for i in range (compare_lineno, block_lineno): block = block + "\n" block = block + token['value'] + "\n" else: # and if we do for i in range (compare_lineno, block_lineno): block = block + "\n" block = block + token['value'] + "\n" else: usefull_token = token usefull_token['signifier'] = str(token['line_number']) + "_syntax_error" usefull_tokens.append(usefull_token) if token['variant'] == 'type': maybe_signifier = peek(index + 2, uncollapsed_tokens) expression = '' line_number = token['line_number'] if maybe_signifier["variant"] == "signifier": name = maybe_signifier['value'] for i in find_token_indicies(name, uncollapsed_tokens): maybe_expression = peek(i + 2, uncollapsed_tokens) if maybe_expression and maybe_expression['variant'] == 'expression': line_number = maybe_expression['line_number'] expression = maybe_expression['value'] elif maybe_expression and maybe_expression['variant'] == 'function_call': expression = find_return_statement(maybe_expression, usefull_tokens.copy()) usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression } usefull_tokens.append(usefull_token) if token['variant'] == 'function_declaration': temp = collect_body(index, uncollapsed_tokens) body = collapse_tokens(temp) usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body} usefull_tokens.append(usefull_token) if token['variant'] == 'function_call': usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']} usefull_tokens.append(usefull_token) if token['variant'] == 'return': for tkn in usefull_tokens.copy(): if token['value'] == tkn['signifier']: usefull_token = {"variant": "return_statement", "signifier": tkn['signifier'], "type": tkn['type'], "line_number": tkn['line_number'], "expression": tkn['expression']} usefull_tokens.append(usefull_token) usefull_tokens.sort(key=operator.itemgetter('line_number')) return usefull_tokens def remove_token(removable: dict, tokens: list) -> list: for index in range(0,len(tokens)): token = tokens[index] if token == removable: del tokens[index] break return tokens collapsed_tokens = collapse_tokens(tokens) for index in range(0, len(collapsed_tokens)): token = collapsed_tokens[index] if token['variant'] == 'function_call': function = coaless_tokens([token['signifier']], token['line_number'],collapsed_tokens.copy(), "function_declaration") collapsed_tokens[index]['function'] = function.pop() params = coaless_tokens(token['params'], token['line_number'],collapsed_tokens.copy(), "variable_declaration") collapsed_tokens[index]['params'] = params minus = 0 for index in range(0, len(collapsed_tokens)): token = collapsed_tokens[index - minus].copy() if token['variant'] == 'function_declaration': for part in token['body']: collapsed_tokens = remove_token(part,collapsed_tokens.copy()) minus = minus + 1 if token['variant'] == 'function_call': for part in token['function']['body']: collapsed_tokens = remove_token(part,collapsed_tokens.copy()) minus = minus + 1 collapsed_tokens = remove_token(token['function'],collapsed_tokens.copy()) minus = minus + 1 print(json.dumps(collapsed_tokens))