parser: implement initial tokeniser

only logging position, errors and non-whitespace elements

Signed-off-by: Alejandro Mery <amery@jpi.io>

parser/lexer_runes.go

@@ -0,0 +1,19 @@
+package parser
+
+import "asciigoat.org/core/lexer"
+
+var (
+	// IsNewLine tells if a rune represents a line break or the start of one
+	IsNewLine = lexer.NewIsIn("\n\r")
+	// IsSpace tells if a rune is considered whitespace by unicode
+	IsSpace = lexer.IsSpace
+	// IsNotNewLine tells if a rune is anything other than line breaks
+	IsNotNewLine = lexer.NewIsNot(IsNewLine)
+	// IsNotSpace tells if a rune is anything other than whitespace
+	IsNotSpace = lexer.NewIsNot(IsSpace)
+)
+
+// IsSpaceNotNewLine indicates a rune is whitespace but not a new line
+func IsSpaceNotNewLine(r rune) bool {
+	return IsSpace(r) && !IsNewLine(r)
+}