package lexer import ( "bytes" "io" "strings" "unicode/utf8" ) const ( // ReadBufferSize indicates the initial buffer size ReadBufferSize = 1 << 7 // 128B // DoublingBufferSizeLimit indicates when we stop doubling // and just add instead DoublingBufferSizeLimit = 1 << 17 // 128KiB ) // implemented interfaces var ( _ io.RuneReader = (*Reader)(nil) ) // Reader is a RuneReader aimed at implementing text parsers type Reader struct { src io.Reader buf []byte off int cursor int } // String returns what's already Read but not yet emitted or discarded func (b *Reader) String() string { return string(b.buf[b.off:b.cursor]) } // Emit returns what's already being Read and discards it afterwards func (b *Reader) Emit() string { s := b.String() b.Discard() return s } // Discard removes from the buffer everything that has been Read func (b *Reader) Discard() { switch { case b.ready() == 0: // reset b.buf = b.buf[:0] b.cursor = 0 b.off = 0 default: // step b.off = b.cursor } } // ready tells how many bytes are ready to decode func (b *Reader) ready() int { return len(b.buf) - b.cursor } // available tells how many free bytes remain at the end of the buffer func (b *Reader) available() int { return cap(b.buf) - len(b.buf) } func (b *Reader) needsBytes(n int) error { for { if b.ready() >= n { // ready return nil } // make room b.prepareBuffer(n - b.ready()) // and read more _, err := b.fill() if err != nil { return err } } } func (b *Reader) rebuffer(size int) { var src, dst []byte if size > cap(b.buf) { // new buffer dst = make([]byte, size) } else { // same buffer dst = b.buf } src = b.buf[b.off:] dst = dst[:len(src)] copy(dst, src) b.cursor -= b.off b.buf = dst b.off = 0 } func (b *Reader) prepareBuffer(n int) { if n > b.available() { needed := len(b.buf) + n - b.off size := cap(b.buf) for size < needed { switch { case size < DoublingBufferSizeLimit: size *= 2 default: size += DoublingBufferSizeLimit } } b.rebuffer(size) } } func (b *Reader) fill() (int, error) { start := len(b.buf) n, err := b.src.Read(b.buf[start:cap(b.buf)]) if n > 0 { b.buf = b.buf[:start+n] } return n, err } // ReadRune reads the next rune func (b *Reader) ReadRune() (rune, int, error) { // we need at least one byte to start count := 1 for { err := b.needsBytes(count) if err != nil { return 0, 0, err } if utf8.FullRune(b.buf[b.cursor:]) { // we have a full rune break } // more count = b.ready() + 1 } // decode rune r, l := utf8.DecodeRune(b.buf[b.cursor:]) return r, l, nil } // NewReader creates a new runes [Reader] using the given [io.Reader] func NewReader(r io.Reader) *Reader { if r == nil { return nil } return &Reader{ src: r, buf: make([]byte, 0, ReadBufferSize), } } // NewReaderBytes creates a new runes [Reader] using the given bytes func NewReaderBytes(b []byte) *Reader { return NewReader(bytes.NewReader(b)) } // NewReaderString creates a new runes [Reader] using the given string func NewReaderString(s string) *Reader { return NewReader(strings.NewReader(s)) }