Refactor a small bit

pass1.py

@@ -5,7 +5,6 @@ tokens = dict()
 with open('./tokens.json', 'r') as json_tokens:
   tokens = json.loads(json_tokens.read())
 
-usefull_tokens = list()
 
 def peek(index: int) -> dict:
     global tokens
@@ -24,15 +23,16 @@ def find_token_indicies(key: str) -> list:
         index = index + 1
     return result
 
-def collect_body(index: int) -> list():
+def collect_body(index: int) -> list:
-    result = list()
+    global tokens
+    uncollapsed_tokens = list()
     token = peek(index + 1)
     if token['variant'] != "operator":
-        return result
+        return uncollapsed_tokens
     maybe_expression = peek(index + 2)
     if maybe_expression['variant'] == 'expression':
-        result.append(token)
+        uncollapsed_tokens.append(token)
-        return result
+        return uncollapsed_tokens
     elif maybe_expression['variant'] == 'start_block':
         index = index + 2
         while True:
@@ -40,34 +40,44 @@ def collect_body(index: int) -> list():
             collectable = peek(index)
             if collectable['variant'] == 'end_block':
                 break
-            result.append(collectable)
+            uncollapsed_tokens.append(collectable)
+    return uncollapsed_tokens
 
-    return result
+def collapse_tokens(uncollapsed_tokens: list) -> list:
+    usefull_tokens = list()
+    for index in range(0, len(uncollapsed_tokens)):
+        token = uncollapsed_tokens[index]
 
+        # This is a declaration
+        if token['variant'] == 'type':
+            maybe_signifier = peek(index + 2)
+            expression = ''
+            line_number = token['line_number']
+            if maybe_signifier["variant"] == "signifier":
+                name = maybe_signifier['value']
+                for i in find_token_indicies(name):
+                    maybe_expression = peek(i + 2)
+                    if maybe_expression['variant'] == 'expression':
+                        line_number = maybe_expression['line_number']
+                        expression = maybe_expression['value']
+                        break
+                usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression } 
+                usefull_tokens.append(usefull_token)
+        if token['variant'] == 'function_declaration':
+            body = collect_body(index)
+            usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body} 
             
-for index in range(0, len(tokens)):
-    token = tokens[index]
-
-    # This is a declaration
-    if token['variant'] == 'type':
-        maybe_signifier = peek(index + 2)
-        expression = ''
-        line_number = token['line_number']
-        if maybe_signifier["variant"] == "signifier":
-            name = maybe_signifier['value']
-            for i in find_token_indicies(name):
-                maybe_expression = peek(i + 2)
-                if maybe_expression['variant'] == 'expression':
-                    line_number = maybe_expression['line_number']
-                    expression = maybe_expression['value']
-                    break
-            usefull_token = { "variant": "variable_declaration", "signifier": name, "type": token['value'], "line_number": line_number, "expression": expression } 
             usefull_tokens.append(usefull_token)
-    if token['variant'] == 'function_declaration':
-        body = collect_body(index)
-        usefull_token = { "variant": "function_declaration", "signifier": token['value'], "line_number": token['line_number'], "params": token['params'] ,"body": body} 
 
-        usefull_tokens.append(usefull_token)
+    usefull_tokens.sort(key=operator.itemgetter('line_number'))
+    return usefull_tokens
+
+collapsed_tokens = collapse_tokens(tokens)
 
-sorted_tokens = usefull_tokens.sort(key=operator.itemgetter('line_number'))
+#for index in range(0, len(collapsed_tokens)):
-print(json.dumps(usefull_tokens))
+#    token = collapsed_tokens[index]
+#    if token['variant'] == 'function_declaration':
+#        old_body = token['body'].copy()
+#        new_body = collapse_tokens(old_body)
+#        collapsed_tokens[index]['body'] = new_body
+print(json.dumps(collapsed_tokens))

prog.mn

@@ -13,8 +13,9 @@ func: hello(float: x) = {
 
 hello(x)
 
-func: good_bye(void: none) = int: getout = 0
+func: good_bye(void: none) = {
-
+    int: getout = 0
+}
 good_bye()
 
 if (x ? 1) {

tokens.json

@@ -1,263 +1 @@
-[
+[{"variant": "type", "value": "float", "line_number": 1}, {"variant": "operator", "value": ":", "line_number": 1}, {"variant": "signifier", "value": "x", "line_number": 1}, {"variant": "operator", "value": "=", "line_number": 1}, {"variant": "expression", "value": "10.1", "line_number": 1}, {"variant": "type", "value": "func", "line_number": 3}, {"variant": "operator", "value": ":", "line_number": 3}, {"variant": "function_declaration", "value": "hello", "params": [{"type": "float", "name": "x"}], "line_number": 3}, {"variant": "operator", "value": "=", "line_number": 3}, {"variant": "start_block", "value": "{", "line_number": 3}, {"variant": "type", "value": "stdout", "line_number": 4}, {"variant": "operator", "value": ":", "line_number": 4}, {"variant": "signifier", "value": "out", "line_number": 4}, {"variant": "operator", "value": "=", "line_number": 4}, {"variant": "expression", "value": "x * 10", "line_number": 4}, {"variant": "end_block", "value": "}", "line_number": 5}, {"variant": "start_block", "value": "{", "line_number": 7}, {"variant": "type", "value": "str", "line_number": 8}, {"variant": "operator", "value": ":", "line_number": 8}, {"variant": "signifier", "value": "name", "line_number": 8}, {"variant": "type", "value": "int", "line_number": 9}, {"variant": "operator", "value": ":", "line_number": 9}, {"variant": "signifier", "value": "var", "line_number": 9}, {"variant": "operator", "value": "=", "line_number": 9}, {"variant": "expression", "value": "1", "line_number": 9}, {"variant": "signifier", "value": "name", "line_number": 10}, {"variant": "operator", "value": "=", "line_number": 10}, {"variant": "expression", "value": "hej hopp", "line_number": 10}, {"variant": "end_block", "value": "}", "line_number": 11}, {"variant": "function_call", "value": "hello", "params": ["x"], "line_number": 14}, {"variant": "type", "value": "func", "line_number": 16}, {"variant": "operator", "value": ":", "line_number": 16}, {"variant": "function_declaration", "value": "good_bye", "params": [{"type": "void", "name": "none"}], "line_number": 16}, {"variant": "operator", "value": "=", "line_number": 16}, {"variant": "start_block", "value": "{", "line_number": 16}, {"variant": "type", "value": "int", "line_number": 17}, {"variant": "operator", "value": ":", "line_number": 17}, {"variant": "signifier", "value": "getout", "line_number": 17}, {"variant": "operator", "value": "=", "line_number": 17}, {"variant": "expression", "value": "0", "line_number": 17}, {"variant": "end_block", "value": "}", "line_number": 18}, {"variant": "function_call", "value": "good_bye", "params": [""], "line_number": 19}, {"variant": "conditional", "value": "if", "condition": "x ? 1", "line_number": 21}, {"variant": "start_block", "value": "{", "line_number": 21}, {"variant": "function_call", "value": "hello", "params": ["x"], "line_number": 22}, {"variant": "end_block", "value": "}", "line_number": 23}, {"variant": "loop", "value": "while", "condition": "x < 1", "line_number": 25}, {"variant": "start_block", "value": "{", "line_number": 25}, {"variant": "signifier", "value": "x", "line_number": 26}, {"variant": "operator", "value": "=", "line_number": 26}, {"variant": "expression", "value": "x -1", "line_number": 26}, {"variant": "function_call", "value": "hello", "params": ["x"], "line_number": 27}, {"variant": "end_block", "value": "}", "line_number": 28}]
-  {
-    "variant": "type",
-    "value": "float",
-    "line_number": 1
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 1
-  },
-  {
-    "variant": "signifier",
-    "value": "x",
-    "line_number": 1
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 1
-  },
-  {
-    "variant": "expression",
-    "value": "10.1",
-    "line_number": 1
-  },
-  {
-    "variant": "type",
-    "value": "func",
-    "line_number": 3
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 3
-  },
-  {
-    "variant": "function_declaration",
-    "value": "hello",
-    "params": [
-      {
-        "type": "float",
-        "name": "x"
-      }
-    ],
-    "line_number": 3
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 3
-  },
-  {
-    "variant": "start_block",
-    "value": "{",
-    "line_number": 3
-  },
-  {
-    "variant": "type",
-    "value": "stdout",
-    "line_number": 4
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 4
-  },
-  {
-    "variant": "signifier",
-    "value": "out",
-    "line_number": 4
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 4
-  },
-  {
-    "variant": "expression",
-    "value": "x * 10",
-    "line_number": 4
-  },
-  {
-    "variant": "end_block",
-    "value": "}",
-    "line_number": 5
-  },
-  {
-    "variant": "start_block",
-    "value": "{",
-    "line_number": 7
-  },
-  {
-    "variant": "type",
-    "value": "str",
-    "line_number": 8
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 8
-  },
-  {
-    "variant": "signifier",
-    "value": "name",
-    "line_number": 8
-  },
-  {
-    "variant": "type",
-    "value": "int",
-    "line_number": 9
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 9
-  },
-  {
-    "variant": "signifier",
-    "value": "var",
-    "line_number": 9
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 9
-  },
-  {
-    "variant": "expression",
-    "value": "1",
-    "line_number": 9
-  },
-  {
-    "variant": "signifier",
-    "value": "name",
-    "line_number": 10
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 10
-  },
-  {
-    "variant": "expression",
-    "value": "hej hopp",
-    "line_number": 10
-  },
-  {
-    "variant": "end_block",
-    "value": "}",
-    "line_number": 11
-  },
-  {
-    "variant": "function_call",
-    "value": "hello",
-    "params": [
-      "x"
-    ],
-    "line_number": 14
-  },
-  {
-    "variant": "type",
-    "value": "func",
-    "line_number": 16
-  },
-  {
-    "variant": "operator",
-    "value": ":",
-    "line_number": 16
-  },
-  {
-    "variant": "function_declaration",
-    "value": "good_bye",
-    "params": [
-      {
-        "type": "void",
-        "name": "none"
-      }
-    ],
-    "line_number": 16
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 16
-  },
-  {
-    "variant": "expression",
-    "value": "int: getout = 0",
-    "line_number": 16
-  },
-  {
-    "variant": "function_call",
-    "value": "good_bye",
-    "params": [
-      ""
-    ],
-    "line_number": 18
-  },
-  {
-    "variant": "conditional",
-    "value": "if",
-    "condition": "x ? 1",
-    "line_number": 20
-  },
-  {
-    "variant": "start_block",
-    "value": "{",
-    "line_number": 20
-  },
-  {
-    "variant": "function_call",
-    "value": "hello",
-    "params": [
-      "x"
-    ],
-    "line_number": 21
-  },
-  {
-    "variant": "end_block",
-    "value": "}",
-    "line_number": 22
-  },
-  {
-    "variant": "loop",
-    "value": "while",
-    "condition": "x < 1",
-    "line_number": 24
-  },
-  {
-    "variant": "start_block",
-    "value": "{",
-    "line_number": 24
-  },
-  {
-    "variant": "signifier",
-    "value": "x",
-    "line_number": 25
-  },
-  {
-    "variant": "operator",
-    "value": "=",
-    "line_number": 25
-  },
-  {
-    "variant": "expression",
-    "value": "x -1",
-    "line_number": 25
-  },
-  {
-    "variant": "function_call",
-    "value": "hello",
-    "params": [
-      "x"
-    ],
-    "line_number": 26
-  },
-  {
-    "variant": "end_block",
-    "value": "}",
-    "line_number": 27
-  }
-]