Parser somewhat working
This commit is contained in:
Micke Nordin
parent
f7b9cd734a
commit
c38f5ffacc
GPG key ID:
0DA0A7A5708FE257
4 changed files with 124 additions and 133 deletions
4
hello.mn
4
hello.mn
|
|
@ -8,3 +8,7 @@ name = hej hopp
|
|||
|
||||
hello(name)
|
||||
|
||||
|
||||
string: micke = Micke Nordin
|
||||
|
||||
hello(micke)
|
||||
|
|
|
|||
33
hello.tokens
33
hello.tokens
|
|
@ -97,5 +97,38 @@
|
|||
"name"
|
||||
],
|
||||
"line_number": 9
|
||||
},
|
||||
{
|
||||
"variant": "type",
|
||||
"value": "string",
|
||||
"line_number": 12
|
||||
},
|
||||
{
|
||||
"variant": "operator",
|
||||
"value": ":",
|
||||
"line_number": 12
|
||||
},
|
||||
{
|
||||
"variant": "signifier",
|
||||
"value": "micke",
|
||||
"line_number": 12
|
||||
},
|
||||
{
|
||||
"variant": "operator",
|
||||
"value": "=",
|
||||
"line_number": 12
|
||||
},
|
||||
{
|
||||
"variant": "expression",
|
||||
"value": "Micke Nordin",
|
||||
"line_number": 12
|
||||
},
|
||||
{
|
||||
"variant": "function_call",
|
||||
"value": "hello",
|
||||
"params": [
|
||||
"micke"
|
||||
],
|
||||
"line_number": 14
|
||||
}
|
||||
]
|
||||
|
|
|
|||
116
parser.py
116
parser.py
|
|
@ -1,12 +1,22 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
|
||||
import operator
|
||||
import sys
|
||||
tokens = dict()
|
||||
with open('./tokens.json', 'r') as json_tokens:
|
||||
tokens = json.loads(json_tokens.read())
|
||||
program = './tokens.json'
|
||||
if len(sys.argv) > 1:
|
||||
program = sys.argv[1]
|
||||
with open(program, 'r') as json_tokens:
|
||||
tokens = json.loads(json_tokens.read())
|
||||
|
||||
def find_token_indicies(key: str) -> list:
|
||||
global tokens
|
||||
|
||||
def peek(index: int, tokens: list) -> dict:
|
||||
result = dict()
|
||||
if index < len(tokens):
|
||||
result = tokens[index]
|
||||
return result
|
||||
|
||||
def find_token_indicies(key: str, tokens: list) -> list:
|
||||
result = list()
|
||||
index = 0
|
||||
for token in tokens:
|
||||
|
|
@ -15,30 +25,78 @@ def find_token_indicies(key: str) -> list:
|
|||
index = index + 1
|
||||
return result
|
||||
|
||||
def peek(index: int) -> dict:
|
||||
global tokens
|
||||
result = dict()
|
||||
if index < len(tokens):
|
||||
result = tokens[index]
|
||||
def collect_body(index: int, tokens: list) -> list:
|
||||
uncollapsed_tokens = list()
|
||||
token = peek(index + 1, tokens)
|
||||
if token['variant'] != "operator":
|
||||
return uncollapsed_tokens
|
||||
maybe_expression = peek(index + 2, tokens)
|
||||
if maybe_expression['variant'] == 'expression':
|
||||
uncollapsed_tokens.append(token)
|
||||
return uncollapsed_tokens
|
||||
elif maybe_expression['variant'] == 'start_block':
|
||||
index = index + 2
|
||||
while True:
|
||||
index = index + 1
|
||||
collectable = peek(index, tokens)
|
||||
if collectable['variant'] == 'end_block':
|
||||
break
|
||||
uncollapsed_tokens.append(collectable)
|
||||
return uncollapsed_tokens
|
||||
|
||||
def coaless_tokens(identifiers, line_number, usefull_tokens, variant) -> list:
|
||||
result = list()
|
||||
for identifier in identifiers:
|
||||
candidate_token = dict()
|
||||
for token in usefull_tokens:
|
||||
if line_number > token['line_number'] and identifier == token['signifier'] and token['variant'] == variant:
|
||||
candidate_token = token
|
||||
if candidate_token:
|
||||
result.append(candidate_token)
|
||||
return result
|
||||
|
||||
current_scope = "main"
|
||||
stack = [{"scope": current_scope}]
|
||||
ast = {current_scope: {"env": stack[-1] }}
|
||||
for index in range(0, len(tokens)):
|
||||
token = tokens[index]
|
||||
if token['variant'] == "start_block":
|
||||
current_scope = "user_" + str(token["line_number"])
|
||||
stack.append({"scope": current_scope} )
|
||||
ast[current_scope] = {}
|
||||
if token['variant'] == "end_block":
|
||||
old_stack = stack.pop()
|
||||
ast[current_scope]["env"] = old_stack
|
||||
current_scope = stack[-1]["scope"]
|
||||
if token['variant'] == "type":
|
||||
maybe_signifier = peek(index + 2)
|
||||
if maybe_signifier["variant"] == "signifier":
|
||||
name = maybe_signifier['value']
|
||||
stack[-1][name] = {"type": token['value']}
|
||||
def collapse_tokens(uncollapsed_tokens: list) -> list:
|
||||
usefull_tokens = list()
|
||||
for index in range(0, len(uncollapsed_tokens)):
|
||||
token = uncollapsed_tokens[index]
|
||||
|
||||
print(json.dumps(ast))
|
||||
# This is a declaration
|
||||
if token['variant'] == 'type':
|
||||
maybe_signifier = peek(index + 2, uncollapsed_tokens)
|
||||
expression = ''
|
||||
line_number = token['line_number']
|
||||
if maybe_signifier["variant"] == "signifier":
|
||||
name = maybe_signifier['value']
|
||||
for i in find_token_indicies(name, uncollapsed_tokens):
|
||||
maybe_expression = peek(i + 2, uncollapsed_tokens)
|
||||
if maybe_expression and maybe_expression['variant'] == 'expression':
|
||||
line_number = maybe_expression['line_number']
|
||||
expression = maybe_expression['value']
|
||||
break
|
||||
usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression }
|
||||
usefull_tokens.append(usefull_token)
|
||||
if token['variant'] == 'function_declaration':
|
||||
temp = collect_body(index, uncollapsed_tokens)
|
||||
body = collapse_tokens(temp)
|
||||
usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body}
|
||||
|
||||
usefull_tokens.append(usefull_token)
|
||||
|
||||
if token['variant'] == 'function_call':
|
||||
usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']}
|
||||
|
||||
usefull_tokens.append(usefull_token)
|
||||
|
||||
usefull_tokens.sort(key=operator.itemgetter('line_number'))
|
||||
return usefull_tokens
|
||||
|
||||
collapsed_tokens = collapse_tokens(tokens)
|
||||
|
||||
for index in range(0, len(collapsed_tokens)):
|
||||
token = collapsed_tokens[index]
|
||||
if token['variant'] == 'function_call':
|
||||
function = coaless_tokens([token['signifier']], token['line_number'],collapsed_tokens.copy(), "function_declaration")
|
||||
collapsed_tokens[index]['function'] = function
|
||||
params = coaless_tokens(token['params'], token['line_number'],collapsed_tokens.copy(), "variable_declaration")
|
||||
collapsed_tokens[index]['params'] = params
|
||||
print(json.dumps(collapsed_tokens))
|
||||
|
|
|
|||
104
pass1.py
104
pass1.py
|
|
@ -1,104 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
import operator
|
||||
import sys
|
||||
tokens = dict()
|
||||
program = './tokens.json'
|
||||
if len(sys.argv) > 1:
|
||||
program = sys.argv[1]
|
||||
with open(program, 'r') as json_tokens:
|
||||
tokens = json.loads(json_tokens.read())
|
||||
|
||||
|
||||
def peek(index: int) -> dict:
|
||||
global tokens
|
||||
result = dict()
|
||||
if index < len(tokens):
|
||||
result = tokens[index]
|
||||
return result
|
||||
|
||||
def find_token_indicies(key: str) -> list:
|
||||
global tokens
|
||||
result = list()
|
||||
index = 0
|
||||
for token in tokens:
|
||||
if token["value"] == key:
|
||||
result.append(index)
|
||||
index = index + 1
|
||||
return result
|
||||
|
||||
def collect_body(index: int) -> list:
|
||||
global tokens
|
||||
uncollapsed_tokens = list()
|
||||
token = peek(index + 1)
|
||||
if token['variant'] != "operator":
|
||||
return uncollapsed_tokens
|
||||
maybe_expression = peek(index + 2)
|
||||
if maybe_expression['variant'] == 'expression':
|
||||
uncollapsed_tokens.append(token)
|
||||
return uncollapsed_tokens
|
||||
elif maybe_expression['variant'] == 'start_block':
|
||||
index = index + 2
|
||||
while True:
|
||||
index = index + 1
|
||||
collectable = peek(index)
|
||||
if collectable['variant'] == 'end_block':
|
||||
break
|
||||
uncollapsed_tokens.append(collectable)
|
||||
return uncollapsed_tokens
|
||||
|
||||
def collect_parameters(params, line_number, usefull_tokens) -> list:
|
||||
result = list()
|
||||
for param in params:
|
||||
candidate_token = dict()
|
||||
for token in usefull_tokens:
|
||||
if line_number > token['line_number'] and param == token['signifier']:
|
||||
candidate_token = token
|
||||
if candidate_token:
|
||||
result.append(candidate_token)
|
||||
return result
|
||||
|
||||
def collapse_tokens(uncollapsed_tokens: list) -> list:
|
||||
usefull_tokens = list()
|
||||
for index in range(0, len(uncollapsed_tokens)):
|
||||
token = uncollapsed_tokens[index]
|
||||
|
||||
# This is a declaration
|
||||
if token['variant'] == 'type':
|
||||
maybe_signifier = peek(index + 2)
|
||||
expression = ''
|
||||
line_number = token['line_number']
|
||||
if maybe_signifier["variant"] == "signifier":
|
||||
name = maybe_signifier['value']
|
||||
for i in find_token_indicies(name):
|
||||
maybe_expression = peek(i + 2)
|
||||
if maybe_expression['variant'] == 'expression':
|
||||
line_number = maybe_expression['line_number']
|
||||
expression = maybe_expression['value']
|
||||
break
|
||||
usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression }
|
||||
usefull_tokens.append(usefull_token)
|
||||
if token['variant'] == 'function_declaration':
|
||||
body = collect_body(index)
|
||||
usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body}
|
||||
|
||||
usefull_tokens.append(usefull_token)
|
||||
|
||||
if token['variant'] == 'function_call':
|
||||
usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']}
|
||||
|
||||
usefull_tokens.append(usefull_token)
|
||||
|
||||
usefull_tokens.sort(key=operator.itemgetter('line_number'))
|
||||
return usefull_tokens
|
||||
|
||||
collapsed_tokens = collapse_tokens(tokens)
|
||||
|
||||
for index in range(0, len(collapsed_tokens)):
|
||||
token = collapsed_tokens[index]
|
||||
if token['variant'] == 'function_call':
|
||||
function = collect_parameters([token['signifier']], token['line_number'],collapsed_tokens.copy())
|
||||
collapsed_tokens[index]['function'] = function
|
||||
params = collect_parameters(token['params'], token['line_number'],collapsed_tokens.copy())
|
||||
collapsed_tokens[index]['params'] = params
|
||||
print(json.dumps(collapsed_tokens))
|
||||
Loading…
Add table
Reference in a new issue