From 418ae6fa0eb8fa584adfccb87fab1a6e531f1e5e Mon Sep 17 00:00:00 2001
From: Devine Lu Linvega <aliceffekt@gmail.com>
Date: Mon, 25 Mar 2024 19:49:04 -0700
Subject: [PATCH] (uxnasm) New tokenizer

---
 src/uxnasm.c | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/uxnasm.c b/src/uxnasm.c
index 2a443b0..333b2f0 100644
--- a/src/uxnasm.c
+++ b/src/uxnasm.c
@@ -456,11 +456,23 @@ resolve(void)
 static int
 assemble(FILE *f)
 {
+	unsigned int buf;
+	char *cptr = token;
 	p.ptr = 0x100;
 	scpy("on-reset", p.scope, 0x40);
-	while(fscanf(f, "%62s", token) == 1)
-		if(slen(token) > 0x3d || !parse(token, f))
-			return error_asm("Invalid token");
+	while(fread(&buf, 1, 1, f)) {
+		char c = (char)buf;
+		if(c < 0x21) {
+			*cptr++ = 0x00;
+			if(token[0])
+				if(!parse(token, f))
+					return 0;
+			cptr = token;
+		} else if(cptr - token < 0x3f)
+			*cptr++ = c;
+		else
+			return error_asm("Token too long");
+	}
 	return resolve();
 }