From 7230a74f494582cbee55f7f8fa81888ccd593d9a Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 29 Aug 2023 01:56:28 +0000 Subject: [PATCH 1/3] lexer: runes.Reader renamed to lexer.Reader Signed-off-by: Alejandro Mery --- lexer/lexer.go | 2 ++ {runes => lexer}/reader.go | 2 +- runes/docs.go | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 lexer/lexer.go rename {runes => lexer}/reader.go (99%) delete mode 100644 runes/docs.go diff --git a/lexer/lexer.go b/lexer/lexer.go new file mode 100644 index 0000000..33b17b6 --- /dev/null +++ b/lexer/lexer.go @@ -0,0 +1,2 @@ +// Package lexer provides basic helpers to implement parsers +package lexer diff --git a/runes/reader.go b/lexer/reader.go similarity index 99% rename from runes/reader.go rename to lexer/reader.go index 1e73048..a36b5a9 100644 --- a/runes/reader.go +++ b/lexer/reader.go @@ -1,4 +1,4 @@ -package runes +package lexer import ( "bytes" diff --git a/runes/docs.go b/runes/docs.go deleted file mode 100644 index ba5c527..0000000 --- a/runes/docs.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package runes helps us work with runes -package runes -- 2.17.1 From edcba80baa85dc90f0e0dd4e93a90ce32f2c859f Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 29 Aug 2023 00:45:52 +0000 Subject: [PATCH 2/3] lexer: fix ReadRune() to actually move the cursor Signed-off-by: Alejandro Mery --- lexer/reader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lexer/reader.go b/lexer/reader.go index a36b5a9..75ea32a 100644 --- a/lexer/reader.go +++ b/lexer/reader.go @@ -153,6 +153,9 @@ func (b *Reader) ReadRune() (rune, int, error) { // decode rune r, l := utf8.DecodeRune(b.buf[b.cursor:]) + // step over + b.cursor += l + return r, l, nil } -- 2.17.1 From 6cca2996caeab7f15e95774625d5116a35f44859 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 28 Aug 2023 23:16:01 +0000 Subject: [PATCH 3/3] lexer: Implement Reader.UnreadRune() and Reader.PeekRune() Signed-off-by: Alejandro Mery --- lexer/reader.go | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/lexer/reader.go b/lexer/reader.go index 75ea32a..7c0aaa3 100644 --- a/lexer/reader.go +++ b/lexer/reader.go @@ -2,6 +2,7 @@ package lexer import ( "bytes" + "errors" "io" "strings" "unicode/utf8" @@ -18,7 +19,14 @@ const ( // implemented interfaces var ( - _ io.RuneReader = (*Reader)(nil) + _ io.RuneReader = (*Reader)(nil) + _ io.RuneScanner = (*Reader)(nil) +) + +var ( + // ErrInvalidUnreadRune indicates UnreadRune() was calls after an + // action other than a successful ReadRune() + ErrInvalidUnreadRune = errors.New("invalid UnreadRune() call") ) // Reader is a RuneReader aimed at implementing text parsers @@ -28,6 +36,8 @@ type Reader struct { buf []byte off int cursor int + + lastRuneSize int } // String returns what's already Read but not yet emitted or discarded @@ -54,6 +64,9 @@ func (b *Reader) Discard() { // step b.off = b.cursor } + + // and prevent UnreadRune() + b.lastRuneSize = -1 } // ready tells how many bytes are ready to decode @@ -139,6 +152,8 @@ func (b *Reader) ReadRune() (rune, int, error) { for { err := b.needsBytes(count) if err != nil { + b.lastRuneSize = -1 + return 0, 0, err } @@ -155,10 +170,34 @@ func (b *Reader) ReadRune() (rune, int, error) { r, l := utf8.DecodeRune(b.buf[b.cursor:]) // step over b.cursor += l + // and remember for UnreadRune() + b.lastRuneSize = l return r, l, nil } +// UnreadRune moves the cursor where it was before the last call to ReadRune +func (b *Reader) UnreadRune() error { + if b.lastRuneSize > 0 { + b.cursor -= b.lastRuneSize + b.lastRuneSize = -1 + return nil + } + + return ErrInvalidUnreadRune +} + +// PeekRune returns information about the next rune without moving the +// cursor +func (b *Reader) PeekRune() (rune, int, error) { + r, l, err := b.ReadRune() + if err != nil { + return r, l, err + } + err = b.UnreadRune() + return r, l, err +} + // NewReader creates a new runes [Reader] using the given [io.Reader] func NewReader(r io.Reader) *Reader { if r == nil { -- 2.17.1