|
|
|
#!/usr/bin/env python3
|
|
|
|
import json
|
|
|
|
import operator
|
|
|
|
import sys
|
|
|
|
tokens = dict()
|
|
|
|
program = './tokens.json'
|
|
|
|
if len(sys.argv) > 1:
|
|
|
|
program = sys.argv[1]
|
|
|
|
with open(program, 'r') as json_tokens:
|
|
|
|
tokens = json.loads(json_tokens.read())
|
|
|
|
|
|
|
|
|
|
|
|
def peek(index: int, tokens: list) -> dict:
|
|
|
|
result = dict()
|
|
|
|
if index < len(tokens):
|
|
|
|
result = tokens[index]
|
|
|
|
return result
|
|
|
|
|
|
|
|
def find_token_indicies(key: str, tokens: list) -> list:
|
|
|
|
result = list()
|
|
|
|
index = 0
|
|
|
|
for token in tokens:
|
|
|
|
if token["value"] == key:
|
|
|
|
result.append(index)
|
|
|
|
index = index + 1
|
|
|
|
return result
|
|
|
|
|
|
|
|
def find_return_statement(maybe_expression, usefull_tokens) -> dict:
|
|
|
|
expression = dict()
|
|
|
|
for token in usefull_tokens:
|
|
|
|
if maybe_expression['value'] == token['signifier'] and token['variant'] == 'function_declaration':
|
|
|
|
for part in token['body']:
|
|
|
|
if part['variant'] == 'return_statement':
|
|
|
|
return part
|
|
|
|
return expression
|
|
|
|
|
|
|
|
def collect_body(index: int, tokens: list) -> list:
|
|
|
|
uncollapsed_tokens = list()
|
|
|
|
token = peek(index + 1, tokens)
|
|
|
|
if token['variant'] != "operator":
|
|
|
|
return uncollapsed_tokens
|
|
|
|
maybe_expression = peek(index + 2, tokens)
|
|
|
|
if maybe_expression['variant'] == 'expression':
|
|
|
|
uncollapsed_tokens.append(token)
|
|
|
|
return uncollapsed_tokens
|
|
|
|
elif maybe_expression['variant'] == 'start_block':
|
|
|
|
index = index + 2
|
|
|
|
while True:
|
|
|
|
index = index + 1
|
|
|
|
collectable = peek(index, tokens)
|
|
|
|
if collectable['variant'] == 'end_block':
|
|
|
|
break
|
|
|
|
uncollapsed_tokens.append(collectable)
|
|
|
|
return uncollapsed_tokens
|
|
|
|
|
|
|
|
def coaless_tokens(identifiers, line_number, usefull_tokens, variant) -> list:
|
|
|
|
result = list()
|
|
|
|
for identifier in identifiers:
|
|
|
|
candidate_token = dict()
|
|
|
|
for token in usefull_tokens:
|
|
|
|
if line_number > token['line_number'] and identifier == token['signifier'] and token['variant'] == variant:
|
|
|
|
candidate_token = token
|
|
|
|
if candidate_token:
|
|
|
|
result.append(candidate_token)
|
|
|
|
return result
|
|
|
|
|
|
|
|
def collapse_tokens(uncollapsed_tokens: list) -> list:
|
|
|
|
usefull_tokens = list()
|
|
|
|
in_block = False
|
|
|
|
block = ""
|
|
|
|
block_index = -1
|
|
|
|
block_lineno = -1
|
|
|
|
for index in range(0, len(uncollapsed_tokens)):
|
|
|
|
token = uncollapsed_tokens[index]
|
|
|
|
if token['variant'] == 'start_block':
|
|
|
|
block_index = index
|
|
|
|
in_block = True
|
|
|
|
block_lineno = token['line_number']
|
|
|
|
if token['variant'] == 'end_block':
|
|
|
|
compare_lineno = block_lineno + 1
|
|
|
|
block_lineno = token['line_number']
|
|
|
|
# Do we need to append new lines
|
|
|
|
for i in range (compare_lineno, block_lineno):
|
|
|
|
block = block + "\n"
|
|
|
|
|
|
|
|
name = "multiline"
|
|
|
|
maybe_variable = peek(block_index - 2, uncollapsed_tokens)
|
|
|
|
if maybe_variable['variant'] == 'signifier':
|
|
|
|
name = maybe_variable['value']
|
|
|
|
usefull_token = { "variant": "variable_declaration", "signifier": name, "type": "string", "line_number": maybe_variable['line_number'], "expression": block }
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
# Reset for next multiline
|
|
|
|
in_block = False
|
|
|
|
block = str()
|
|
|
|
block_index = -1
|
|
|
|
block_lineno = -1
|
|
|
|
|
|
|
|
if token['variant'] == 'syntax_error':
|
|
|
|
if in_block:
|
|
|
|
compare_lineno = block_lineno + 1
|
|
|
|
block_lineno = token['line_number']
|
|
|
|
# This is if we don't have a block allready
|
|
|
|
if block == "":
|
|
|
|
# Do we need to prepend empty lines?
|
|
|
|
for i in range (compare_lineno, block_lineno):
|
|
|
|
block = block + "\n"
|
|
|
|
block = block + token['value'] + "\n"
|
|
|
|
else:
|
|
|
|
# and if we do
|
|
|
|
for i in range (compare_lineno, block_lineno):
|
|
|
|
block = block + "\n"
|
|
|
|
block = block + token['value'] + "\n"
|
|
|
|
|
|
|
|
else:
|
|
|
|
usefull_token = token
|
|
|
|
usefull_token['signifier'] = str(token['line_number']) + "_syntax_error"
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
if token['variant'] == 'type':
|
|
|
|
maybe_signifier = peek(index + 2, uncollapsed_tokens)
|
|
|
|
expression = ''
|
|
|
|
line_number = token['line_number']
|
|
|
|
if maybe_signifier["variant"] == "signifier":
|
|
|
|
name = maybe_signifier['value']
|
|
|
|
for i in find_token_indicies(name, uncollapsed_tokens):
|
|
|
|
maybe_expression = peek(i + 2, uncollapsed_tokens)
|
|
|
|
if maybe_expression and maybe_expression['variant'] == 'expression':
|
|
|
|
line_number = maybe_expression['line_number']
|
|
|
|
expression = maybe_expression['value']
|
|
|
|
elif maybe_expression and maybe_expression['variant'] == 'function_call':
|
|
|
|
expression = find_return_statement(maybe_expression, usefull_tokens.copy())
|
|
|
|
usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression }
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
if token['variant'] == 'function_declaration':
|
|
|
|
temp = collect_body(index, uncollapsed_tokens)
|
|
|
|
body = collapse_tokens(temp)
|
|
|
|
usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body}
|
|
|
|
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
|
|
|
|
if token['variant'] == 'function_call':
|
|
|
|
usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']}
|
|
|
|
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
if token['variant'] == 'return':
|
|
|
|
for tkn in usefull_tokens.copy():
|
|
|
|
if token['value'] == tkn['signifier']:
|
|
|
|
usefull_token = {"variant": "return_statement", "signifier": tkn['signifier'], "type": tkn['type'], "line_number": tkn['line_number'], "expression": tkn['expression']}
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
|
|
|
|
usefull_tokens.sort(key=operator.itemgetter('line_number'))
|
|
|
|
return usefull_tokens
|
|
|
|
|
|
|
|
def remove_token(removable: dict, tokens: list) -> list:
|
|
|
|
for index in range(0,len(tokens)):
|
|
|
|
token = tokens[index]
|
|
|
|
if token == removable:
|
|
|
|
del tokens[index]
|
|
|
|
break
|
|
|
|
return tokens
|
|
|
|
|
|
|
|
collapsed_tokens = collapse_tokens(tokens)
|
|
|
|
|
|
|
|
for index in range(0, len(collapsed_tokens)):
|
|
|
|
token = collapsed_tokens[index]
|
|
|
|
if token['variant'] == 'function_call':
|
|
|
|
function = coaless_tokens([token['signifier']], token['line_number'],collapsed_tokens.copy(), "function_declaration")
|
|
|
|
collapsed_tokens[index]['function'] = function.pop()
|
|
|
|
params = coaless_tokens(token['params'], token['line_number'],collapsed_tokens.copy(), "variable_declaration")
|
|
|
|
collapsed_tokens[index]['params'] = params
|
|
|
|
|
|
|
|
minus = 0
|
|
|
|
for index in range(0, len(collapsed_tokens)):
|
|
|
|
token = collapsed_tokens[index - minus].copy()
|
|
|
|
if token['variant'] == 'function_declaration':
|
|
|
|
for part in token['body']:
|
|
|
|
collapsed_tokens = remove_token(part,collapsed_tokens.copy())
|
|
|
|
minus = minus + 1
|
|
|
|
if token['variant'] == 'function_call':
|
|
|
|
for part in token['function']['body']:
|
|
|
|
collapsed_tokens = remove_token(part,collapsed_tokens.copy())
|
|
|
|
minus = minus + 1
|
|
|
|
collapsed_tokens = remove_token(token['function'],collapsed_tokens.copy())
|
|
|
|
minus = minus + 1
|
|
|
|
|
|
|
|
|
|
|
|
print(json.dumps(collapsed_tokens))
|