parser: add internal []Token queue to the Parser

Signed-off-by: Alejandro Mery <amery@jpi.io>

parser/lexer_queue.go

@@ -0,0 +1,44 @@
+package parser
+
+import "log"
+
+func (p *Parser) push(tok Token) {
+	n := len(p.queue)
+	p.queue = append(p.queue, tok)
+
+	log.Printf("queue[%v]: %s", n, tok)
+}
+
+func (p *Parser) pushString(typ TokenType) {
+	s := p.src.Emit()
+
+	el := Token{
+		Type:     typ,
+		Value:    s,
+		Position: p.pos,
+	}
+
+	p.pos.StepN(len(s))
+
+	p.push(el)
+}
+
+// stepLine discards the data and moves the position
+// to the next line
+func (p *Parser) stepLine() {
+	p.src.Discard()
+	p.pos.StepLine()
+}
+
+// stepRune discards the data and moves the position
+// on rune forward on the same line
+func (p *Parser) stepRune() {
+	p.src.Discard()
+	p.pos.Step()
+}
+
+func (p *Parser) stepString() string {
+	s := p.src.Emit()
+	p.pos.StepN(len(s))
+	return s
+}