|
|
|
@ -57,11 +57,49 @@ def coaless_tokens(identifiers, line_number, usefull_tokens, variant) -> list:
|
|
|
|
|
|
|
|
|
|
def collapse_tokens(uncollapsed_tokens: list) -> list:
|
|
|
|
|
usefull_tokens = list()
|
|
|
|
|
in_block = False
|
|
|
|
|
block = str()
|
|
|
|
|
block_index = -1
|
|
|
|
|
block_lineno = -1
|
|
|
|
|
for index in range(0, len(uncollapsed_tokens)):
|
|
|
|
|
token = uncollapsed_tokens[index]
|
|
|
|
|
if token['variant'] == 'start_block':
|
|
|
|
|
block_index = index
|
|
|
|
|
in_block = True
|
|
|
|
|
block_lineno = token['line_number']
|
|
|
|
|
if token['variant'] == 'end_block':
|
|
|
|
|
in_block = False
|
|
|
|
|
name = "multiline"
|
|
|
|
|
maybe_variable = peek(block_index - 2, uncollapsed_tokens)
|
|
|
|
|
if maybe_variable['variant'] == 'signifier':
|
|
|
|
|
name = maybe_variable['value']
|
|
|
|
|
usefull_token = { "variant": "variable_declaration", "signifier": name, "type": "string", "line_number": maybe_variable['line_number'], "expression": block }
|
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
|
block = str()
|
|
|
|
|
block_index = -1
|
|
|
|
|
block_lineno = -1
|
|
|
|
|
|
|
|
|
|
# This is a declaration
|
|
|
|
|
if token['variant'] == 'syntax_error':
|
|
|
|
|
if in_block:
|
|
|
|
|
old_lineno = block_lineno
|
|
|
|
|
block_lineno = token['line_number']
|
|
|
|
|
if block:
|
|
|
|
|
for i in range (old_lineno, block_lineno -1):
|
|
|
|
|
block = block + "\n"
|
|
|
|
|
block = block + token['value']
|
|
|
|
|
else:
|
|
|
|
|
if old_lineno +1 != block_lineno:
|
|
|
|
|
for i in range (old_lineno, block_lineno -1):
|
|
|
|
|
block = block + "\n"
|
|
|
|
|
else:
|
|
|
|
|
block = block + "\n"
|
|
|
|
|
|
|
|
|
|
block = block + token['value']
|
|
|
|
|
for i in range (old_lineno, block_lineno -1):
|
|
|
|
|
block = block + "\n"
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
usefull_token = token
|
|
|
|
|
usefull_token['signifier'] = str(token['line_number']) + "_syntax_error"
|
|
|
|
|
usefull_tokens.append(usefull_token)
|
|
|
|
|