2 Commits

Author SHA1 Message Date
amery ed04fe873b build-sys: use local asciigoat.org/core [DO-NOT-MERGE]
Signed-off-by: Alejandro Mery <amery@jpi.io>
2023-08-30 01:54:45 +00:00
amery 2b9996da7c parser: implement initial tokeniser
only logging position, errors and non-whitespace elements

Signed-off-by: Alejandro Mery <amery@jpi.io>
2023-08-30 01:53:23 +00:00
+11 -9
View File
@@ -22,15 +22,10 @@ func (p *Parser) lexStart() (lexer.StateFn, error) {
return nil, err
case IsNewLine(r):
// new line
p.src.UnreadRune()
p.lexNewLine()
p.src.Discard()
p.pos.StepLine()
p.lexNewLine(r)
case IsSpace(r):
// whitespace
p.src.Discard()
p.pos.Step()
p.lexWhitespace()
default:
// token
p.src.UnreadRune()
@@ -49,9 +44,13 @@ func (p *Parser) lexToken() (lexer.StateFn, error) {
return p.lexStart, nil
}
func (p *Parser) lexNewLine() {
func (p *Parser) lexWhitespace() {
p.src.Discard()
p.pos.Step()
}
func (p *Parser) lexNewLine(r1 rune) {
// r1 is warrantied to be either \n or \r
r1, _, _ := p.src.ReadRune()
r2, _, err := p.src.ReadRune()
switch {
@@ -66,4 +65,7 @@ func (p *Parser) lexNewLine() {
// LN
p.src.UnreadRune()
}
p.src.Discard()
p.pos.StepLine()
}