Unmarshal: WIP

Signed-off-by: Alejandro Mery <amery@jpi.io>

go.mod

@@ -5,7 +5,7 @@ go 1.19
 replace asciigoat.org/core => ../core
 
 require (
-	asciigoat.org/core v0.3.2
+	asciigoat.org/core v0.3.4
 	github.com/mgechev/revive v1.3.3
 )
 

parser/lexer.go

@@ -1,6 +1,10 @@
 package parser
 
-import "asciigoat.org/core/lexer"
+import (
+	"log"
+
+	"asciigoat.org/core/lexer"
+)
 
 // Run parses the source
 func (p *Parser) Run() error {
@@ -15,44 +19,81 @@ func (p *Parser) lexStart() (lexer.StateFn, error) {
 		r, _, err := p.src.ReadRune()
 		switch {
 		case err != nil:
+			// read error
 			return p.emitError("", err)
-		case r == '\n':
+		case IsNewLine(r):
 			// new line
-			r2, _, err := p.src.ReadRune()
-			switch {
-			case err == nil && r2 == '\r':
-				// MAC new line
-			case err == nil:
-				// UNIX new line
-				p.src.UnreadRune()
-			}
-
+			p.lexNewLine(r)
 			p.src.Discard()
 			p.pos.StepLine()
-		case r == '\r':
-			// new line
-			r2, _, err := p.src.ReadRune()
-			switch {
-			case err == nil && r2 == '\n':
-				// DOS new line
-			case err == nil:
-				// incomplete CRLN
-				p.src.UnreadRune()
-				return p.emitInvalidRune(r2)
-			}
-
-			p.src.Discard()
-			p.pos.StepLine()
-		case p.IsSpace(r):
+		case IsSpace(r):
 			// whitespace
 			p.src.Discard()
 			p.pos.Step()
-		case p.IsCommentStart(r):
+		case IsCommentStart(r):
 			// switch to comment lexer
 			p.src.UnreadRune()
 			return p.lexComment, nil
+		case IsSectionStart(r):
+			// section
+			p.src.Discard()
+			p.pos.Step()
+			return p.lexSectionStart, nil
+		default:
+			// entry
+			p.src.UnreadRune()
+			return p.lexEntryStart, nil
 		}
 	}
 }
 
-func (p *Parser) lexComment() (lexer.StateFn, error)
+func (p *Parser) lexNewLine(r1 rune) {
+	// r1 is warrantied to be either \n or \r
+	r2, _, err := p.src.ReadRune()
+
+	switch r1 {
+	case '\r':
+		switch {
+		case r2 == '\n':
+			// CR LN
+		case err == nil:
+			// CR
+			p.src.UnreadRune()
+		default:
+			// CR EOF
+		}
+	case '\n':
+		switch {
+		case r2 == '\r':
+			// LN CR
+		case err == nil:
+			// LN
+			p.src.UnreadRune()
+		default:
+			// LN EOF
+		}
+	default:
+		panic("unreachable")
+	}
+}
+
+func (p *Parser) lexComment() (lexer.StateFn, error) {
+	// until the end of the line
+	p.src.AcceptAll(IsNotNewLine)
+
+	err := p.OnComment(p.pos, p.src.Emit())
+	return p.lexStart, err
+}
+
+func (*Parser) lexSectionStart() (lexer.StateFn, error) { return nil, nil }
+func (*Parser) lexEntryStart() (lexer.StateFn, error)   { return nil, nil }
+
+func (p *Parser) lexToken() (lexer.StateFn, error) {
+	p.src.AcceptAll(IsNotSpace)
+
+	s := p.src.Emit()
+	log.Printf("%s: %s: %q", p.pos, "token", s)
+	p.pos.StepN(len(s))
+
+	return p.lexStart, nil
+}

parser/lexer_runes.go

@@ -1,17 +1,20 @@
 package parser
 
-import "unicode"
+import "asciigoat.org/core/lexer"
 
-func (*Parser) IsSpace(r rune) bool {
-	return unicode.IsSpace(r)
-}
+var (
+	// IsNewLine tells if a rune represents a line break or the start of one
+	IsNewLine = lexer.NewIsIn("\n\r")
+	// IsSpace tells if a rune is considered whitespace by unicode
+	IsSpace = lexer.IsSpace
+	// IsNotNewLine tells if a rune is anything other than line breaks
+	IsNotNewLine = lexer.NewIsNot(IsNewLine)
+	// IsNotSpace tells if a rune is anything other than whitespace
+	IsNotSpace = lexer.NewIsNot(IsSpace)
 
-func (*Parser) IsCommentStart(r rune) bool {
-	// TODO: use Parser options
-	switch r {
-	case ';', '#':
-		return true
-	default:
-		return false
-	}
+	IsCommentStart = lexer.NewIsIn(";#")
+)
+
+func IsSectionStart(r rune) bool {
+	return r == '['
 }