parser: implement initial tokeniser

only logging position, errors and non-whitespace elements

Signed-off-by: Alejandro Mery <amery@jpi.io>

go.mod

@@ -5,7 +5,7 @@ go 1.19
 replace asciigoat.org/core => ../core
 
 require (
-	asciigoat.org/core v0.3.4
+	asciigoat.org/core v0.3.2
 	github.com/mgechev/revive v1.3.3
 )
 

parser/lexer.go

@@ -8,8 +8,6 @@ import (
 
 // Run parses the source
 func (p *Parser) Run() error {
-	p.setDefaults()
-
 	p.pos.Reset()
 	return lexer.Run(p.lexStart)
 }
@@ -20,74 +18,27 @@ func (p *Parser) lexStart() (lexer.StateFn, error) {
 		switch {
 		case err != nil:
 			// read error
-			return p.emitError("", err)
+			log.Printf("%s: %s: %s", p.pos, "error", err)
+			return nil, err
 		case IsNewLine(r):
 			// new line
-			p.lexNewLine(r)
+			p.src.UnreadRune()
+			p.lexNewLine()
+
 			p.src.Discard()
 			p.pos.StepLine()
 		case IsSpace(r):
 			// whitespace
 			p.src.Discard()
 			p.pos.Step()
-		case IsCommentStart(r):
-			// switch to comment lexer
-			p.src.UnreadRune()
-			return p.lexComment, nil
-		case IsSectionStart(r):
-			// section
-			p.src.Discard()
-			p.pos.Step()
-			return p.lexSectionStart, nil
 		default:
-			// entry
+			// token
 			p.src.UnreadRune()
-			return p.lexEntryStart, nil
+			return p.lexToken, nil
 		}
 	}
 }
 
-func (p *Parser) lexNewLine(r1 rune) {
-	// r1 is warrantied to be either \n or \r
-	r2, _, err := p.src.ReadRune()
-
-	switch r1 {
-	case '\r':
-		switch {
-		case r2 == '\n':
-			// CR LN
-		case err == nil:
-			// CR
-			p.src.UnreadRune()
-		default:
-			// CR EOF
-		}
-	case '\n':
-		switch {
-		case r2 == '\r':
-			// LN CR
-		case err == nil:
-			// LN
-			p.src.UnreadRune()
-		default:
-			// LN EOF
-		}
-	default:
-		panic("unreachable")
-	}
-}
-
-func (p *Parser) lexComment() (lexer.StateFn, error) {
-	// until the end of the line
-	p.src.AcceptAll(IsNotNewLine)
-
-	err := p.OnComment(p.pos, p.src.Emit())
-	return p.lexStart, err
-}
-
-func (*Parser) lexSectionStart() (lexer.StateFn, error) { return nil, nil }
-func (*Parser) lexEntryStart() (lexer.StateFn, error)   { return nil, nil }
-
 func (p *Parser) lexToken() (lexer.StateFn, error) {
 	p.src.AcceptAll(IsNotSpace)
 
@@ -97,3 +48,22 @@ func (p *Parser) lexToken() (lexer.StateFn, error) {
 
 	return p.lexStart, nil
 }
+
+func (p *Parser) lexNewLine() {
+	// r1 is warrantied to be either \n or \r
+	r1, _, _ := p.src.ReadRune()
+	r2, _, err := p.src.ReadRune()
+
+	switch {
+	case r1 == '\r' && r2 == '\n':
+		// CR LN
+	case r1 == '\r' && err == nil:
+		// CR
+		p.src.UnreadRune()
+	case r2 == '\r':
+		// LN CR
+	case err == nil:
+		// LN
+		p.src.UnreadRune()
+	}
+}

parser/lexer_error.go

@@ -1,19 +0,0 @@
-package parser
-
-import "asciigoat.org/core/lexer"
-
-func (p *Parser) emitError(content string, err error) (lexer.StateFn, error) {
-	err2 := p.OnError(p.pos, content, err)
-	switch {
-	case err2 != nil:
-		// return wrapped error
-		return nil, err2
-	default:
-		// return original error
-		return nil, err
-	}
-}
-
-func (p *Parser) emitInvalidRune(r rune) (lexer.StateFn, error) {
-	return p.emitError(string([]rune{r}), lexer.ErrUnacceptableRune)
-}

parser/lexer_runes.go

@@ -11,10 +11,4 @@ var (
 	IsNotNewLine = lexer.NewIsNot(IsNewLine)
 	// IsNotSpace tells if a rune is anything other than whitespace
 	IsNotSpace = lexer.NewIsNot(IsSpace)
-
-	IsCommentStart = lexer.NewIsIn(";#")
 )
-
-func IsSectionStart(r rune) bool {
-	return r == '['
-}

parser/parser.go

@@ -12,42 +12,6 @@ type Parser struct {
 	src *lexer.Reader
 
 	pos lexer.Position
-
-	OnSection func(pos lexer.Position, name, subname string, hasSubname bool) error
-	OnField   func(pos lexer.Position, key, value string) error
-	OnComment func(pos lexer.Position, comment string) error
-	OnError   func(pos lexer.Position, content string, err error) error
-}
-
-func defaultOnSection(_ lexer.Position, _, _ string, _ bool) error { return nil }
-func defaultOnField(_ lexer.Position, _, _ string) error           { return nil }
-func defaultOnComment(_ lexer.Position, _ string) error            { return nil }
-
-func defaultOnError(pos lexer.Position, content string, err error) error {
-	return &lexer.Error{
-		Line:    pos.Line,
-		Column:  pos.Column,
-		Content: content,
-		Err:     err,
-	}
-}
-
-func (p *Parser) setDefaults() {
-	if p.OnSection == nil {
-		p.OnSection = defaultOnSection
-	}
-
-	if p.OnField == nil {
-		p.OnField = defaultOnField
-	}
-
-	if p.OnComment == nil {
-		p.OnComment = defaultOnComment
-	}
-
-	if p.OnError == nil {
-		p.OnError = defaultOnError
-	}
 }
 
 // NewParser creates a dosini-style parser using