diff --git a/lexer/lexer.go b/lexer/lexer.go new file mode 100644 index 0000000..33b17b6 --- /dev/null +++ b/lexer/lexer.go @@ -0,0 +1,2 @@ +// Package lexer provides basic helpers to implement parsers +package lexer diff --git a/runes/reader.go b/lexer/reader.go similarity index 76% rename from runes/reader.go rename to lexer/reader.go index 1e73048..7c0aaa3 100644 --- a/runes/reader.go +++ b/lexer/reader.go @@ -1,7 +1,8 @@ -package runes +package lexer import ( "bytes" + "errors" "io" "strings" "unicode/utf8" @@ -18,7 +19,14 @@ const ( // implemented interfaces var ( - _ io.RuneReader = (*Reader)(nil) + _ io.RuneReader = (*Reader)(nil) + _ io.RuneScanner = (*Reader)(nil) +) + +var ( + // ErrInvalidUnreadRune indicates UnreadRune() was calls after an + // action other than a successful ReadRune() + ErrInvalidUnreadRune = errors.New("invalid UnreadRune() call") ) // Reader is a RuneReader aimed at implementing text parsers @@ -28,6 +36,8 @@ type Reader struct { buf []byte off int cursor int + + lastRuneSize int } // String returns what's already Read but not yet emitted or discarded @@ -54,6 +64,9 @@ func (b *Reader) Discard() { // step b.off = b.cursor } + + // and prevent UnreadRune() + b.lastRuneSize = -1 } // ready tells how many bytes are ready to decode @@ -139,6 +152,8 @@ func (b *Reader) ReadRune() (rune, int, error) { for { err := b.needsBytes(count) if err != nil { + b.lastRuneSize = -1 + return 0, 0, err } @@ -153,9 +168,36 @@ func (b *Reader) ReadRune() (rune, int, error) { // decode rune r, l := utf8.DecodeRune(b.buf[b.cursor:]) + // step over + b.cursor += l + // and remember for UnreadRune() + b.lastRuneSize = l + return r, l, nil } +// UnreadRune moves the cursor where it was before the last call to ReadRune +func (b *Reader) UnreadRune() error { + if b.lastRuneSize > 0 { + b.cursor -= b.lastRuneSize + b.lastRuneSize = -1 + return nil + } + + return ErrInvalidUnreadRune +} + +// PeekRune returns information about the next rune without moving the +// cursor +func (b *Reader) PeekRune() (rune, int, error) { + r, l, err := b.ReadRune() + if err != nil { + return r, l, err + } + err = b.UnreadRune() + return r, l, err +} + // NewReader creates a new runes [Reader] using the given [io.Reader] func NewReader(r io.Reader) *Reader { if r == nil { diff --git a/runes/docs.go b/runes/docs.go deleted file mode 100644 index ba5c527..0000000 --- a/runes/docs.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package runes helps us work with runes -package runes