parser: implement initial tokeniser

only logging position, errors and non-whitespace elements

Signed-off-by: Alejandro Mery <amery@jpi.io>

go.mod

@@ -2,7 +2,10 @@ module asciigoat.org/ini
 
 go 1.19
 
-require github.com/mgechev/revive v1.3.3
+require (
+	asciigoat.org/core v0.3.6
+	github.com/mgechev/revive v1.3.3
+)
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect

go.sum

@@ -1,3 +1,5 @@
+asciigoat.org/core v0.3.6 h1:b1vL090OxylmSOwLQryjrmC8FhhCtktMyeJSy1e6LwI=
+asciigoat.org/core v0.3.6/go.mod h1:tXj+JUutxRbcO40ZQRuUVaZ4rnYz1kAZ0nblisV8u74=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/chavacava/garif v0.0.0-20230608123814-4bd63c2919ab h1:5JxePczlyGAtj6R1MUEFZ/UFud6FfsOejq7xLC2ZIb0=

parser/lexer.go

@@ -0,0 +1,78 @@
+package parser
+
+import (
+	"log"
+
+	"asciigoat.org/core/lexer"
+)
+
+// Run parses the source
+func (p *Parser) Run() error {
+	p.pos.Reset()
+	return lexer.Run(p.lexStart)
+}
+
+func (p *Parser) lexStart() (lexer.StateFn, error) {
+	for {
+		r, _, err := p.src.ReadRune()
+		switch {
+		case err != nil:
+			// read error
+			log.Printf("%s: %s: %s", p.pos, "error", err)
+			return nil, err
+		case IsNewLine(r):
+			// new line
+			p.lexNewLine(r)
+			p.src.Discard()
+			p.pos.StepLine()
+		case IsSpace(r):
+			// whitespace
+			p.src.Discard()
+			p.pos.Step()
+		default:
+			// token
+			p.src.UnreadRune()
+			return p.lexToken, nil
+		}
+	}
+}
+
+func (p *Parser) lexToken() (lexer.StateFn, error) {
+	p.src.AcceptAll(IsNotSpace)
+
+	s := p.src.Emit()
+	log.Printf("%s: %s: %q", p.pos, "token", s)
+	p.pos.StepN(len(s))
+
+	return p.lexStart, nil
+}
+
+func (p *Parser) lexNewLine(r1 rune) {
+	// r1 is warrantied to be either \n or \r
+	r2, _, err := p.src.ReadRune()
+
+	switch r1 {
+	case '\r':
+		switch {
+		case r2 == '\n':
+			// CR LN
+		case err == nil:
+			// CR
+			p.src.UnreadRune()
+		default:
+			// CR EOF
+		}
+	case '\n':
+		switch {
+		case r2 == '\r':
+			// LN CR
+		case err == nil:
+			// LN
+			p.src.UnreadRune()
+		default:
+			// LN EOF
+		}
+	default:
+		panic("unreachable")
+	}
+}

parser/lexer_runes.go

@@ -0,0 +1,14 @@
+package parser
+
+import "asciigoat.org/core/lexer"
+
+var (
+	// IsNewLine tells if a rune represents a line break or the start of one
+	IsNewLine = lexer.NewIsIn("\n\r")
+	// IsSpace tells if a rune is considered whitespace by unicode
+	IsSpace = lexer.IsSpace
+	// IsNotNewLine tells if a rune is anything other than line breaks
+	IsNotNewLine = lexer.NewIsNot(IsNewLine)
+	// IsNotSpace tells if a rune is anything other than whitespace
+	IsNotSpace = lexer.NewIsNot(IsSpace)
+)

parser/parser.go

@@ -0,0 +1,27 @@
+// Package parser parses dosini-style files
+package parser
+
+import (
+	"io"
+
+	"asciigoat.org/core/lexer"
+)
+
+// Parser parses a dosini-style document
+type Parser struct {
+	src *lexer.Reader
+
+	pos lexer.Position
+}
+
+// NewParser creates a dosini-style parser using
+// an [io.Reader] as source
+func NewParser(r io.Reader) *Parser {
+	if r == nil {
+		return nil
+	}
+
+	return &Parser{
+		src: lexer.NewReader(r),
+	}
+}