ebnf/token: Add initial TokenType

ebnf/token/tokentype.go

@@ -0,0 +1,20 @@
+package token
+
+// types of Token
+type TokenType int
+
+const (
+	TokenError	TokenType = iota + 1
+	TokenEOF
+)
+
+func (typ TokenType) String() string {
+	switch typ {
+	case TokenError:
+		return "ERROR"
+	case TokenEOF:
+		return "EOF"
+	default:
+		return "UNDEFINED"
+	}
+}

ebnf/token/tokentype_test.go

@@ -0,0 +1,25 @@
+package token
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestTokenTypeToString(t *testing.T) {
+	var foo TokenType
+
+	for _, o := range []struct {
+		typ TokenType
+		str string
+	}{
+		{foo, "UNDEFINED"},
+		{TokenError, "ERROR"},
+		{TokenEOF, "EOF"},
+		{1234, "UNDEFINED"},
+	} {
+		str := fmt.Sprintf("%s", o.typ)
+		if str != o.str {
+			t.Errorf("TokenType:%v stringified as %s instead of %s.", int(o.typ), str, o.str)
+		}
+	}
+}