parser: add initial Parser emitting non-whitespace tokens

Signed-off-by: Alejandro Mery <amery@jpi.io>

parser/lexer.go

@@ -0,0 +1,67 @@
+package parser
+
+import "asciigoat.org/core/lexer"
+
+// Run parses the source
+func (p *Parser) Run() error {
+	p.setDefaults()
+	p.pos.Reset()
+
+	return lexer.Run(p.lexStart)
+}
+
+func (p *Parser) lexStart() (lexer.StateFn, error) {
+	for {
+		r, _, err := p.src.ReadRune()
+		switch {
+		case err != nil:
+			return p.emitError("", err)
+		case IsNewLine(r):
+			// new line
+			p.lexMoreNewLine(r)
+			p.stepLine()
+		case IsSpace(r):
+			// whitespace
+			p.stepRune()
+		default:
+			p.src.UnreadRune()
+			return p.lexToken, nil
+		}
+	}
+}
+
+func (p *Parser) lexMoreNewLine(r1 rune) {
+	// r1 is warrantied to be either '\r' or '\n'
+	r2, _, err := p.src.ReadRune()
+	switch r1 {
+	case '\n':
+		switch {
+		case r2 == '\r':
+			// LN CR
+		case err == nil:
+			// LN
+			p.src.UnreadRune()
+		default:
+			// LN EOF
+		}
+	case '\r':
+		switch {
+		case r2 == '\n':
+			// CR LN
+		case err == nil:
+			// CR
+			p.src.UnreadRune()
+		default:
+			// CR EOF
+		}
+	default:
+		panic("unreachable")
+	}
+}
+
+func (p *Parser) lexToken() (lexer.StateFn, error) {
+	p.src.AcceptAll(IsNotSpace)
+
+	err := p.emitString(TokenUnknown)
+	return p.lexStart, err
+}