From 4cfda08fe9a2244beb7af4df435db97e46f2dfff Mon Sep 17 00:00:00 2001
From: Devine Lu Linvega <aliceffekt@gmail.com>
Date: Thu, 4 Apr 2024 09:29:36 -0700
Subject: [PATCH] Starting tokenizer

---
 src/modal.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 test.modal  |  5 +--
 2 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/src/modal.c b/src/modal.c
index 6b84083..138fed7 100644
--- a/src/modal.c
+++ b/src/modal.c
@@ -1,20 +1,92 @@
 #include <stdio.h>
 
+/* dict */
+
+static char dict[0x8000], *next = dict;
+
+static char *
+save(char *s, char c)
+{
+	char *o = next;
+	while((*next++ = *s++) && *s)
+		;
+	*next++ = c;
+	return o;
+}
+
+static char *
+push(char *s, char c)
+{
+	char *d = dict;
+	for(d = dict; d < next; d++) {
+		char *ss = s, *dd = d, a, b;
+		while((a = *dd++) == (b = *ss++))
+			if(!a && !b) return d;
+	}
+	return save(s, c);
+}
+
+/* rule */
+
+typedef struct {
+	char *a;
+	char *b;
+} Rule;
+
+static int rules_len;
+static Rule rules[0x100];
+
+static void
+addrule(FILE *f)
+{
+	char c;
+	printf("Add rule: ");
+	while(f && fread(&c, 1, 1, f) && c && c != 0xa) {
+		printf("%c", c);
+	}
+	printf("\n");
+}
+
+/* program */
+
+static char program[0x1000];
+
+static void
+addprogram(FILE *f){
+	char c;
+	printf("Add program: ");
+	while(f && fread(&c, 1, 1, f) && c && c != 0xa) {
+		printf("%c", c);
+	}
+	printf("\n");
+}
+
+static int phase;
+
+static void
+tokenize(char *t, FILE *f)
+{
+	if(t[0] == '<' && t[1] == '>') {
+		addrule(f);
+		return;
+	}
+	addprogram(f);
+}
+
 static int
 walk(FILE *f)
 {
-	char c, token[0x40], *cptr = token;
+	char c, token[0x40], *tokptr;
+	tokptr = token;
 	while(f && fread(&c, 1, 1, f)) {
-		if(c < 0x21) {
-			*cptr++ = 0x00;
-			printf("> %s\n", token);
-			cptr = token;
-		} else if(cptr - token < 0x3f)
-			*cptr++ = c;
+		if(c < 0x21)
+			*tokptr++ = 0x00, tokenize(token, f), tokptr = token;
+		else if(tokptr - token < 0x3f)
+			*tokptr++ = c;
 		else
 			return printf("Token too long: %s\n", token);
 	}
-	*cptr++ = 0;
+	*tokptr++ = 0x00, tokenize(token, f), tokptr = token;
 	return 1;
 }
 
diff --git a/test.modal b/test.modal
index d6d2fc2..4ad83e9 100644
--- a/test.modal
+++ b/test.modal
@@ -1,3 +1,4 @@
-<> (hello) (bye)
+<> (dup ?x) (?x ?x done)
+<> hello bye
 
-hello world
\ No newline at end of file
+(dup hello)
\ No newline at end of file