Merge pull request 'lexer: introduce a Position (Line, Column) handler' (#8)

Reviewed-on: #8

Makefile

@@ -0,0 +1,50 @@
+.PHONY: all clean generate fmt
+.PHONY: tidy get build test up
+
+GO ?= go
+GOFMT ?= gofmt
+GOFMT_FLAGS = -w -l -s
+GOGENERATE_FLAGS = -v
+
+GOPATH ?= $(shell $(GO) env GOPATH)
+GOBIN ?= $(GOPATH)/bin
+
+TMPDIR ?= $(CURDIR)/.tmp
+TOOLSDIR = $(CURDIR)/tools
+
+REVIVE_CONF ?= $(TOOLSDIR)/revive.toml
+REVIVE_RUN_ARGS ?= -config $(REVIVE_CONF) -formatter friendly
+REVIVE ?= $(GO) run -v github.com/mgechev/revive
+
+V = 0
+Q = $(if $(filter 1,$V),,@)
+M = $(shell if [ "$$(tput colors 2> /dev/null || echo 0)" -ge 8 ]; then printf "\033[34;1m▶\033[0m"; else printf "▶"; fi)
+
+all: get generate tidy build
+
+clean: ; $(info $(M) cleaning…)
+	rm -rf $(TMPDIR)
+
+fmt: ; $(info $(M) reformatting sources…)
+	$Q find . -name '*.go' | xargs -r $(GOFMT) $(GOFMT_FLAGS)
+
+tidy: | fmt ; $(info $(M) tidying up…)
+	$Q $(GO) mod tidy
+	$Q $(GO) vet ./...
+	$Q $(REVIVE) $(REVIVE_RUN_ARGS) ./...
+
+get: ; $(info $(M) downloading dependencies…)
+	$Q $(GO) get -v -tags tools ./...
+
+build: ; $(info $(M) building…)
+	$Q $(GO) build -v ./...
+
+test: ; $(info $(M) building…)
+	$Q $(GO) test ./...
+
+up: ; $(info $(M) updating dependencies…)
+	$Q $(GO) get -u -v ./...
+	$Q $(GO) mod tidy
+
+generate: ; $(info $(M) generating data…)
+	$Q git grep -l '^//go:generate' | sort -uV | xargs -r -n1 $(GO) generate $(GOGENERATE_FLAGS)

go.mod

@@ -1,3 +1,21 @@
 module asciigoat.org/core
 
 go 1.19
+
+require github.com/mgechev/revive v1.3.3
+
+require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/chavacava/garif v0.0.0-20230608123814-4bd63c2919ab // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/tools v0.12.0 // indirect
+)

go.sum

@@ -0,0 +1,47 @@
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/chavacava/garif v0.0.0-20230608123814-4bd63c2919ab h1:5JxePczlyGAtj6R1MUEFZ/UFud6FfsOejq7xLC2ZIb0=
+github.com/chavacava/garif v0.0.0-20230608123814-4bd63c2919ab/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
+github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 h1:zpIH83+oKzcpryru8ceC6BxnoG8TBrhgAvRg8obzup0=
+github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
+github.com/mgechev/revive v1.3.3 h1:GUWzV3g185agbHN4ZdaQvR6zrLVYTUSA2ktvIinivK0=
+github.com/mgechev/revive v1.3.3/go.mod h1:NhpOtVtDbjYNDj697eDUBTobijCDHQKar4HDKc0TuTo=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
+golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

lexer/error.go

@@ -0,0 +1,45 @@
+package lexer
+
+import (
+	"fmt"
+	"strings"
+)
+
+var (
+	_ error = (*Error)(nil)
+)
+
+// Error represents a generic parsing error
+type Error struct {
+	Filename string
+	Line     int
+	Column   int
+
+	Content string
+	Err     error
+}
+
+func (err Error) Error() string {
+	var s []string
+
+	switch {
+	case err.Line > 0 || err.Column > 0:
+		s = append(s, fmt.Sprintf("%s:%v:%v", err.Filename, err.Line, err.Column))
+	case err.Filename != "":
+		s = append(s, err.Filename)
+	}
+
+	if err.Err != nil {
+		s = append(s, err.Err.Error())
+	}
+
+	if err.Content != "" {
+		s = append(s, fmt.Sprintf("%q", err.Content))
+	}
+
+	return strings.Join(s, ": ")
+}
+
+func (err Error) Unwrap() error {
+	return err.Err
+}

lexer/lexer.go

@@ -0,0 +1,2 @@
+// Package lexer provides basic helpers to implement parsers
+package lexer

lexer/position.go

@@ -0,0 +1,66 @@
+package lexer
+
+import "fmt"
+
+// Position indicates a line and column pair on a file.
+// Counting starts at 1.
+type Position struct {
+	Line   int
+	Column int
+}
+
+// String generates a pretty "(Line, Column)"" representation of the Position
+func (p Position) String() string {
+	if p.Line == 0 {
+		p.Reset()
+	}
+
+	return fmt.Sprintf("(%v, %v)", p.Line, p.Column)
+}
+
+// GoString generates a string representation of the Position for %#v usage
+func (p Position) GoString() string {
+	if p.Line == 0 {
+		p.Reset()
+	}
+
+	return fmt.Sprintf("lexer.Position{%v, %v}", p.Line, p.Column)
+}
+
+// Reset places a position at (1,1)
+func (p *Position) Reset() {
+	p.Line, p.Column = 1, 1
+}
+
+// Step moves the column one place
+func (p *Position) Step() {
+	if p.Line == 0 {
+		p.Reset()
+	}
+
+	p.Column++
+}
+
+// StepN moves the column N places forward
+func (p *Position) StepN(n int) {
+	if p.Line == 0 {
+		p.Reset()
+	}
+
+	switch {
+	case n > 0:
+		p.Column += n
+	default:
+		panic(fmt.Errorf("invalid %v increment", n))
+	}
+}
+
+// StepLine moves position to the start of the next line
+func (p *Position) StepLine() {
+	if p.Line == 0 {
+		p.Reset()
+	}
+
+	p.Line++
+	p.Column = 1
+}

lexer/reader.go

@@ -1,7 +1,8 @@
-package runes
+package lexer
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"strings"
 	"unicode/utf8"
@@ -18,7 +19,14 @@ const (
 
 // implemented interfaces
 var (
-	_ io.RuneReader = (*Reader)(nil)
+	_ io.RuneReader  = (*Reader)(nil)
+	_ io.RuneScanner = (*Reader)(nil)
+)
+
+var (
+	// ErrInvalidUnreadRune indicates UnreadRune() was calls after an
+	// action other than a successful ReadRune()
+	ErrInvalidUnreadRune = errors.New("invalid UnreadRune() call")
 )
 
 // Reader is a RuneReader aimed at implementing text parsers
@@ -28,6 +36,8 @@ type Reader struct {
 	buf    []byte
 	off    int
 	cursor int
+
+	lastRuneSize int
 }
 
 // String returns what's already Read but not yet emitted or discarded
@@ -54,6 +64,9 @@ func (b *Reader) Discard() {
 		// step
 		b.off = b.cursor
 	}
+
+	// and prevent UnreadRune()
+	b.lastRuneSize = -1
 }
 
 // ready tells how many bytes are ready to decode
@@ -139,6 +152,8 @@ func (b *Reader) ReadRune() (rune, int, error) {
 	for {
 		err := b.needsBytes(count)
 		if err != nil {
+			b.lastRuneSize = -1
+
 			return 0, 0, err
 		}
 
@@ -153,9 +168,36 @@ func (b *Reader) ReadRune() (rune, int, error) {
 
 	// decode rune
 	r, l := utf8.DecodeRune(b.buf[b.cursor:])
+	// step over
+	b.cursor += l
+	// and remember for UnreadRune()
+	b.lastRuneSize = l
+
 	return r, l, nil
 }
 
+// UnreadRune moves the cursor where it was before the last call to ReadRune
+func (b *Reader) UnreadRune() error {
+	if b.lastRuneSize > 0 {
+		b.cursor -= b.lastRuneSize
+		b.lastRuneSize = -1
+		return nil
+	}
+
+	return ErrInvalidUnreadRune
+}
+
+// PeekRune returns information about the next rune without moving the
+// cursor
+func (b *Reader) PeekRune() (rune, int, error) {
+	r, l, err := b.ReadRune()
+	if err != nil {
+		return r, l, err
+	}
+	err = b.UnreadRune()
+	return r, l, err
+}
+
 // NewReader creates a new runes [Reader] using the given [io.Reader]
 func NewReader(r io.Reader) *Reader {
 	if r == nil {

readcloser.go

@@ -0,0 +1,64 @@
+package core
+
+import (
+	"bytes"
+	"io"
+	"io/fs"
+	"strings"
+)
+
+// ReadCloser adds a Close() to Readers without one
+type ReadCloser struct {
+	r io.Reader
+}
+
+// Read passes the Read() call to the underlying [io.Reader]
+// and fail if it was Closed()
+func (rc *ReadCloser) Read(b []byte) (int, error) {
+	switch {
+	case rc.r != nil:
+		return rc.r.Read(b)
+	default:
+		return 0, fs.ErrClosed
+	}
+}
+
+// Close attempts to Close the underlying [io.Reader], or
+// remove it if it doesn't support Close() and fail
+// if closed twice
+func (rc *ReadCloser) Close() error {
+	switch {
+	case rc.r != nil:
+		rc.r = nil
+		return nil
+	default:
+		return fs.ErrClosed
+	}
+}
+
+// NewReadCloser wraps a [io.Reader] to satisfy
+// [io.ReadCloser] if needed
+func NewReadCloser(r io.Reader) io.ReadCloser {
+	switch p := r.(type) {
+	case io.ReadCloser:
+		return p
+	case nil:
+		return nil
+	default:
+		return &ReadCloser{
+			r: r,
+		}
+	}
+}
+
+// NewReadCloserBytes wraps a bytes slice to implement
+// a [io.ReadCloser]
+func NewReadCloserBytes(b []byte) io.ReadCloser {
+	return NewReadCloser(bytes.NewReader(b))
+}
+
+// NewReadCloserString wraps a string to implement
+// a [io.ReadCloser]
+func NewReadCloserString(s string) io.ReadCloser {
+	return NewReadCloser(strings.NewReader(s))
+}

runes/docs.go

@@ -1,2 +0,0 @@
-// Package runes helps us work with runes
-package runes

tools/revive.toml

@@ -0,0 +1,35 @@
+ignoreGeneratedHeader = false
+severity = "error"
+confidence = 0.8
+errorCode = 1
+warningCode = 0
+enableAllRules = true
+
+[rule.function-length]
+  arguments = [40,0]
+  severity = "warning"
+[rule.function-result-limit]
+  arguments = [3]
+[rule.argument-limit]
+  arguments = [5]
+[rule.cognitive-complexity]
+  arguments = [7]
+[rule.cyclomatic]
+  arguments = [10]
+[rule.line-length-limit]
+  arguments = [100]
+  severity = "warning"
+[rule.comment-spacings]
+  severity = "warning"
+[rule.empty-lines]
+  severity = "warning"
+
+# Disabled rules
+[rule.max-public-structs]
+  disabled = true
+[rule.file-header]
+  disabled = true
+[rule.add-constant]
+  disabled = true
+[rule.banned-characters]
+  disabled = true

tools/tools.go

@@ -0,0 +1,7 @@
+//go:build tools
+
+package tools
+
+import (
+	_ "github.com/mgechev/revive"
+)