2 Commits

Author SHA1 Message Date
amery c87cc3fe89 build-sys: use local asciigoat.org/core [DO-NOT-MERGE]
Signed-off-by: Alejandro Mery <amery@jpi.io>
2023-08-30 02:08:05 +01:00
amery 6f10144b94 parser: implement initial tokeniser
only logging position, errors and non-whitespace elements

Signed-off-by: Alejandro Mery <amery@jpi.io>
2023-08-30 02:06:31 +01:00
+9 -11
View File
@@ -22,10 +22,15 @@ func (p *Parser) lexStart() (lexer.StateFn, error) {
return nil, err
case IsNewLine(r):
// new line
p.lexNewLine(r)
p.src.UnreadRune()
p.lexNewLine()
p.src.Discard()
p.pos.StepLine()
case IsSpace(r):
// whitespace
p.lexWhitespace()
p.src.Discard()
p.pos.Step()
default:
// token
p.src.UnreadRune()
@@ -44,13 +49,9 @@ func (p *Parser) lexToken() (lexer.StateFn, error) {
return p.lexStart, nil
}
func (p *Parser) lexWhitespace() {
p.src.Discard()
p.pos.Step()
}
func (p *Parser) lexNewLine(r1 rune) {
func (p *Parser) lexNewLine() {
// r1 is warrantied to be either \n or \r
r1, _, _ := p.src.ReadRune()
r2, _, err := p.src.ReadRune()
switch {
@@ -65,7 +66,4 @@ func (p *Parser) lexNewLine(r1 rune) {
// LN
p.src.UnreadRune()
}
p.src.Discard()
p.pos.StepLine()
}