Parser somewhat working

hello.mn

@@ -8,3 +8,7 @@ name = hej hopp
 
 hello(name)
 
+
+string: micke = Micke Nordin
+
+hello(micke)

hello.tokens

@@ -97,5 +97,38 @@
       "name"
     ],
     "line_number": 9
+  },
+  {
+    "variant": "type",
+    "value": "string",
+    "line_number": 12
+  },
+  {
+    "variant": "operator",
+    "value": ":",
+    "line_number": 12
+  },
+  {
+    "variant": "signifier",
+    "value": "micke",
+    "line_number": 12
+  },
+  {
+    "variant": "operator",
+    "value": "=",
+    "line_number": 12
+  },
+  {
+    "variant": "expression",
+    "value": "Micke Nordin",
+    "line_number": 12
+  },
+  {
+    "variant": "function_call",
+    "value": "hello",
+    "params": [
+      "micke"
+    ],
+    "line_number": 14
   }
 ]

parser.py

@@ -1,12 +1,22 @@
 #!/usr/bin/env python3
 import json
-
+import operator
+import sys
 tokens = dict()
-with open('./tokens.json', 'r') as json_tokens:
+program = './tokens.json'
+if len(sys.argv) > 1:
+    program = sys.argv[1]
+with open(program, 'r') as json_tokens:
   tokens = json.loads(json_tokens.read())
 
-def find_token_indicies(key: str) -> list:
+
-    global tokens
+def peek(index: int, tokens: list) -> dict:
+    result = dict()
+    if index < len(tokens):
+      result = tokens[index]
+    return result
+
+def find_token_indicies(key: str, tokens: list) -> list:
     result = list()
     index = 0
     for token in tokens:
@@ -15,30 +25,78 @@ def find_token_indicies(key: str) -> list:
         index = index + 1
     return result
 
-def peek(index: int) -> dict:
+def collect_body(index: int, tokens: list) -> list:
-    global tokens
+    uncollapsed_tokens = list()
-    result = dict()
+    token = peek(index + 1, tokens)
-    if index < len(tokens):
+    if token['variant'] != "operator":
-        result = tokens[index]
+        return uncollapsed_tokens
+    maybe_expression = peek(index + 2, tokens)
+    if maybe_expression['variant'] == 'expression':
+        uncollapsed_tokens.append(token)
+        return uncollapsed_tokens
+    elif maybe_expression['variant'] == 'start_block':
+        index = index + 2
+        while True:
+            index = index + 1
+            collectable = peek(index, tokens)
+            if collectable['variant'] == 'end_block':
+                break
+            uncollapsed_tokens.append(collectable)
+    return uncollapsed_tokens
+
+def coaless_tokens(identifiers, line_number, usefull_tokens, variant) -> list:
+    result = list()
+    for identifier in identifiers:
+        candidate_token = dict()
+        for token in usefull_tokens:
+            if line_number > token['line_number'] and identifier == token['signifier'] and token['variant'] == variant:
+                candidate_token = token
+        if candidate_token:
+            result.append(candidate_token)
     return result
 
-current_scope = "main"
+def collapse_tokens(uncollapsed_tokens: list) -> list:
-stack = [{"scope": current_scope}] 
+    usefull_tokens = list()
-ast = {current_scope: {"env": stack[-1] }}
+    for index in range(0, len(uncollapsed_tokens)):
-for index in range(0, len(tokens)):
+        token = uncollapsed_tokens[index]
-    token = tokens[index]
+
-    if token['variant'] == "start_block":
+        # This is a declaration
-        current_scope = "user_" + str(token["line_number"])
+        if token['variant'] == 'type':
-        stack.append({"scope": current_scope} )
+            maybe_signifier = peek(index + 2, uncollapsed_tokens)
-        ast[current_scope] = {}
+            expression = ''
-    if token['variant'] == "end_block":
+            line_number = token['line_number']
-        old_stack = stack.pop()
-        ast[current_scope]["env"] = old_stack
-        current_scope = stack[-1]["scope"]
-    if token['variant'] == "type":
-        maybe_signifier = peek(index + 2)
             if maybe_signifier["variant"] == "signifier":
                 name = maybe_signifier['value']
-            stack[-1][name] = {"type": token['value']}
+                for i in find_token_indicies(name, uncollapsed_tokens):
+                    maybe_expression = peek(i + 2, uncollapsed_tokens)
+                    if maybe_expression and maybe_expression['variant'] == 'expression':
+                        line_number = maybe_expression['line_number']
+                        expression = maybe_expression['value']
+                        break
+                usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression } 
+                usefull_tokens.append(usefull_token)
+        if token['variant'] == 'function_declaration':
+            temp = collect_body(index, uncollapsed_tokens)
+            body = collapse_tokens(temp)
+            usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body} 
+            
+            usefull_tokens.append(usefull_token)
+
+        if token['variant'] == 'function_call':
+            usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']}
+
+            usefull_tokens.append(usefull_token)
+
+    usefull_tokens.sort(key=operator.itemgetter('line_number'))
+    return usefull_tokens
+
+collapsed_tokens = collapse_tokens(tokens)
 
-print(json.dumps(ast))
+for index in range(0, len(collapsed_tokens)):
+    token = collapsed_tokens[index]
+    if token['variant'] == 'function_call':
+        function = coaless_tokens([token['signifier']], token['line_number'],collapsed_tokens.copy(), "function_declaration")
+        collapsed_tokens[index]['function'] = function
+        params = coaless_tokens(token['params'], token['line_number'],collapsed_tokens.copy(), "variable_declaration")
+        collapsed_tokens[index]['params'] = params
+print(json.dumps(collapsed_tokens))

pass1.py

@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-import json
-import operator
-import sys
-tokens = dict()
-program = './tokens.json'
-if len(sys.argv) > 1:
-    program = sys.argv[1]
-with open(program, 'r') as json_tokens:
-  tokens = json.loads(json_tokens.read())
-
-
-def peek(index: int) -> dict:
-    global tokens
-    result = dict()
-    if index < len(tokens):
-      result = tokens[index]
-    return result
-
-def find_token_indicies(key: str) -> list:
-    global tokens
-    result = list()
-    index = 0
-    for token in tokens:
-        if token["value"] == key:
-            result.append(index)
-        index = index + 1
-    return result
-
-def collect_body(index: int) -> list:
-    global tokens
-    uncollapsed_tokens = list()
-    token = peek(index + 1)
-    if token['variant'] != "operator":
-        return uncollapsed_tokens
-    maybe_expression = peek(index + 2)
-    if maybe_expression['variant'] == 'expression':
-        uncollapsed_tokens.append(token)
-        return uncollapsed_tokens
-    elif maybe_expression['variant'] == 'start_block':
-        index = index + 2
-        while True:
-            index = index + 1
-            collectable = peek(index)
-            if collectable['variant'] == 'end_block':
-                break
-            uncollapsed_tokens.append(collectable)
-    return uncollapsed_tokens
-
-def collect_parameters(params, line_number, usefull_tokens) -> list:
-    result = list()
-    for param in params:
-        candidate_token = dict()
-        for token in usefull_tokens:
-            if line_number > token['line_number'] and param == token['signifier']:
-                candidate_token = token
-        if candidate_token:
-            result.append(candidate_token)
-    return result
-
-def collapse_tokens(uncollapsed_tokens: list) -> list:
-    usefull_tokens = list()
-    for index in range(0, len(uncollapsed_tokens)):
-        token = uncollapsed_tokens[index]
-
-        # This is a declaration
-        if token['variant'] == 'type':
-            maybe_signifier = peek(index + 2)
-            expression = ''
-            line_number = token['line_number']
-            if maybe_signifier["variant"] == "signifier":
-                name = maybe_signifier['value']
-                for i in find_token_indicies(name):
-                    maybe_expression = peek(i + 2)
-                    if maybe_expression['variant'] == 'expression':
-                        line_number = maybe_expression['line_number']
-                        expression = maybe_expression['value']
-                        break
-                usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression } 
-                usefull_tokens.append(usefull_token)
-        if token['variant'] == 'function_declaration':
-            body = collect_body(index)
-            usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body} 
-            
-            usefull_tokens.append(usefull_token)
-
-        if token['variant'] == 'function_call':
-            usefull_token = {"variant": "function_call", "signifier": token['value'], "line_number": token['line_number'], "params": token['params']}
-
-            usefull_tokens.append(usefull_token)
-
-    usefull_tokens.sort(key=operator.itemgetter('line_number'))
-    return usefull_tokens
-
-collapsed_tokens = collapse_tokens(tokens)
-
-for index in range(0, len(collapsed_tokens)):
-    token = collapsed_tokens[index]
-    if token['variant'] == 'function_call':
-        function = collect_parameters([token['signifier']], token['line_number'],collapsed_tokens.copy())
-        collapsed_tokens[index]['function'] = function
-        params = collect_parameters(token['params'], token['line_number'],collapsed_tokens.copy())
-        collapsed_tokens[index]['params'] = params
-print(json.dumps(collapsed_tokens))