move confyg into x

Signed-off-by: Xe <me@christine.website>

29 files changed, 1780 insertions, 14 deletions

diff --git a/internal/confyg/LICENSE b/internal/confyg/LICENSE
new file mode 100644
index 0000000..6a66aea
--- /dev/null
+++ b/internal/confyg/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/internal/confyg/README.md b/internal/confyg/README.md
new file mode 100644
index 0000000..c904c25
--- /dev/null
+++ b/internal/confyg/README.md
@@ -0,0 +1,108 @@
+# confyg
+
+A suitably generic form of the Go module configuration file parser.
+
+[![GoDoc](https://godoc.org/within.website/confyg?status.svg)](https://godoc.org/within.website/confyg)
+
+Usage is simple:
+
+```go
+type server struct {
+	port string
+	keys *crypto.Keypair
+	db   *storm.DB
+}
+
+func (s *server) Allow(verb string, block bool) bool {
+	switch verb {
+	case "port":
+		return !block
+	case "dbfile":
+		return !block
+	case "keys":
+		return !block
+	}
+
+	return false
+}
+
+func (s *server) Read(errs *bytes.Buffer, fs *confyg.FileSyntax, line *confyg.Line, verb string, args []string) {
+	switch verb {
+	case "port":
+		_, err := strconv.Atoi(args[0])
+		if err != nil {
+			fmt.Fprintf(errs, "%s:%d value is not a number: %s: %v\n", fs.Name, line.Start.Line, args[0], err)
+			return
+		}
+
+		s.port = args[0]
+
+	case "dbfile":
+		dbFile := args[0][1 : len(args[0])-1] // shuck off quotes
+
+		db, err := storm.Open(dbFile)
+		if err != nil {
+			fmt.Fprintf(errs, "%s:%d failed to open storm database: %s: %v\n", fs.Name, line.Start.Line, args[0], err)
+			return
+		}
+
+		s.db = db
+
+	case "keys":
+		kp := &crypto.Keypair{}
+
+		pubk, err := hex.DecodeString(args[0])
+		if err != nil {
+			fmt.Fprintf(errs, "%s:%d invalid public key: %v\n", fs.Name, line.Start.Line, err)
+			return
+		}
+
+		privk, err := hex.DecodeString(args[1])
+		if err != nil {
+			fmt.Fprintf(errs, "%s:%d invalid private key: %v\n", fs.Name, line.Start.Line, err)
+			return
+		}
+
+		copy(kp.Public[:], pubk[0:32])
+		copy(kp.Private[:], privk[0:32])
+
+		s.keys = kp
+	}
+}
+
+var (
+	configFile = flag.String("cfg", "./apig.cfg", "apig config file location")
+)
+
+func main() {
+	flag.Parse()
+
+	data, err := ioutil.ReadFile(*configFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	s := &server{}
+	_, err = confyg.Parse(*configFile, data, s, s)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_ = s
+}
+```
+
+Or use [`flagconfyg`](https://godoc.org/within.website/confyg/flagconfyg):
+
+```go
+var (
+  config = flag.Config("cfg", "", "if set, configuration file to load (see https://github.com/Xe/x/blob/master/docs/man/flagconfyg.5)")
+)
+
+func main() {
+  flag.Parse()
+  
+  if *config != "" {
+    flagconfyg.CmdParse(*config)
+  }
+}
diff --git a/internal/confyg/allower.go b/internal/confyg/allower.go
new file mode 100644
index 0000000..422aec4
--- /dev/null
+++ b/internal/confyg/allower.go
@@ -0,0 +1,19 @@
+package confyg
+
+// Allower defines if a given verb and block combination is valid for
+// configuration parsing.
+//
+// If this is intended to be a statement-like verb, block should be set
+// to false. If this is intended to be a block-like verb, block should
+// be set to true.
+type Allower interface {
+	Allow(verb string, block bool) bool
+}
+
+// AllowerFunc implements Allower for inline definitions.
+type AllowerFunc func(verb string, block bool) bool
+
+// Allow implements Allower.
+func (a AllowerFunc) Allow(verb string, block bool) bool {
+	return a(verb, block)
+}
diff --git a/internal/confyg/allower_test.go b/internal/confyg/allower_test.go
new file mode 100644
index 0000000..1cae7ac
--- /dev/null
+++ b/internal/confyg/allower_test.go
@@ -0,0 +1,48 @@
+package confyg
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestAllower(t *testing.T) {
+	al := AllowerFunc(func(verb string, block bool) bool {
+		switch verb {
+		case "project":
+			if block {
+				return false
+			}
+
+			return true
+		}
+
+		return false
+	})
+
+	cases := []struct {
+		verb  string
+		block bool
+		want  bool
+	}{
+		{
+			verb:  "project",
+			block: false,
+			want:  true,
+		},
+		{
+			verb:  "nonsense",
+			block: true,
+			want:  false,
+		},
+	}
+
+	for _, cs := range cases {
+		t.Run(fmt.Sprint(cs), func(t *testing.T) {
+			result := al.Allow(cs.verb, cs.block)
+
+			if result != cs.want {
+				t.Fatalf("wanted Allow(%q, %v) == %v, got: %v", cs.verb, cs.block, cs.want, result)
+			}
+		})
+	}
+}
diff --git a/internal/confyg/flagconfyg/flagconfyg.go b/internal/confyg/flagconfyg/flagconfyg.go
new file mode 100644
index 0000000..fd43efe
--- /dev/null
+++ b/internal/confyg/flagconfyg/flagconfyg.go
@@ -0,0 +1,80 @@
+// Package flagconfyg is a hack around confyg. This will blindly convert config
+// verbs to flag values.
+package flagconfyg
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"io/ioutil"
+	"strings"
+
+	"within.website/confyg"
+	"within.website/ln"
+)
+
+// CmdParse is a quick wrapper for command usage. It explodes on errors.
+func CmdParse(ctx context.Context, path string) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return
+	}
+
+	err = Parse(path, data, flag.CommandLine)
+	if err != nil {
+		ln.Error(ctx, err)
+		return
+	}
+}
+
+// Parse parses the config file in the given file by name, bytes data and into
+// the given flagset.
+func Parse(name string, data []byte, fs *flag.FlagSet) error {
+	lineRead := func(errs *bytes.Buffer, fs_ *confyg.FileSyntax, line *confyg.Line, verb string, args []string) {
+		err := fs.Set(verb, strings.Join(args, " "))
+		if err != nil {
+			errs.WriteString(err.Error())
+		}
+	}
+
+	_, err := confyg.Parse(name, data, confyg.ReaderFunc(lineRead), confyg.AllowerFunc(allower))
+	return err
+}
+
+func allower(verb string, block bool) bool {
+	return true
+}
+
+// Dump turns a flagset's values into a configuration file.
+func Dump(fs *flag.FlagSet) []byte {
+	result := &confyg.FileSyntax{
+		Name: fs.Name(),
+		Comments: confyg.Comments{
+			Before: []confyg.Comment{
+				{
+					Token: "// generated from " + fs.Name() + " flags",
+				}, {},
+			},
+		},
+		Stmt: []confyg.Expr{},
+	}
+
+	fs.Visit(func(fl *flag.Flag) {
+		commentTokens := []string{"//", fl.Usage}
+
+		l := &confyg.Line{
+			Comments: confyg.Comments{
+				Suffix: []confyg.Comment{
+					{
+						Token: strings.Join(commentTokens, " "),
+					},
+				},
+			},
+			Token: []string{fl.Name, fl.Value.String()},
+		}
+
+		result.Stmt = append(result.Stmt, l)
+	})
+
+	return confyg.Format(result)
+}
diff --git a/internal/confyg/flagconfyg/flagconfyg_test.go b/internal/confyg/flagconfyg/flagconfyg_test.go
new file mode 100644
index 0000000..767a699
--- /dev/null
+++ b/internal/confyg/flagconfyg/flagconfyg_test.go
@@ -0,0 +1,49 @@
+package flagconfyg
+
+import (
+	"flag"
+	"testing"
+)
+
+func TestFlagConfyg(t *testing.T) {
+	fs := flag.NewFlagSet("test", flag.PanicOnError)
+	sc := fs.String("subscribe", "", "to pewdiepie")
+	us := fs.String("unsubscribe", "all the time", "from t-series")
+
+	const configFile = `subscribe pewdiepie
+
+unsubscribe (
+  t-series
+)`
+
+	err := Parse("test.cfg", []byte(configFile), fs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if *sc != "pewdiepie" {
+		t.Errorf("wanted subscribe->pewdiepie, got: %s", *sc)
+	}
+
+	if *us != "t-series" {
+		t.Errorf("wanted unsubscribe->t-series, got: %s", *us)
+	}
+}
+
+func TestDump(t *testing.T) {
+	fs := flag.NewFlagSet("h", flag.PanicOnError)
+	fs.String("test-string", "some value", "fill this in pls")
+	fs.Bool("test-bool", false, "also fill this in pls")
+
+	err := fs.Parse([]string{"-test-string=foo", "-test-bool"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data := Dump(fs)
+
+	err = Parse("h.cfg", data, fs)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/internal/confyg/map_output.go b/internal/confyg/map_output.go
new file mode 100644
index 0000000..5358f99
--- /dev/null
+++ b/internal/confyg/map_output.go
@@ -0,0 +1,18 @@
+package confyg
+
+import (
+	"bytes"
+	"strings"
+)
+
+// MapConfig is a simple wrapper around a map.
+type MapConfig map[string][]string
+
+// Allow accepts everything.
+func (mc MapConfig) Allow(verb string, block bool) bool {
+	return true
+}
+
+func (mc MapConfig) Read(errs *bytes.Buffer, fs *FileSyntax, line *Line, verb string, args []string) {
+	mc[verb] = append(mc[verb], strings.Join(args, " "))
+}
diff --git a/internal/confyg/map_output_test.go b/internal/confyg/map_output_test.go
new file mode 100644
index 0000000..84012f5
--- /dev/null
+++ b/internal/confyg/map_output_test.go
@@ -0,0 +1,26 @@
+package confyg
+
+import "testing"
+
+func TestMapConfig(t *testing.T) {
+	mc := MapConfig{}
+
+	const configFile = `subscribe pewdiepie
+
+unsubscribe (
+  t-series
+)`
+
+	_, err := Parse("test.cfg", []byte(configFile), mc, mc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if mc["subscribe"][0] != "pewdiepie" {
+		t.Errorf("wanted subscribe->pewdiepie, got: %s", mc["subscribe"][0])
+	}
+
+	if mc["unsubscribe"][0] != "t-series" {
+		t.Errorf("wanted unsubscribe->t-series, got: %s", mc["unsubscribe"][0])
+	}
+}
diff --git a/internal/confyg/print.go b/internal/confyg/print.go
new file mode 100644
index 0000000..68a71ee
--- /dev/null
+++ b/internal/confyg/print.go
@@ -0,0 +1,164 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Module file printer.
+
+package confyg
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+func Format(f *FileSyntax) []byte {
+	pr := &printer{}
+	pr.file(f)
+	return pr.Bytes()
+}
+
+// A printer collects the state during printing of a file or expression.
+type printer struct {
+	bytes.Buffer           // output buffer
+	comment      []Comment // pending end-of-line comments
+	margin       int       // left margin (indent), a number of tabs
+}
+
+// printf prints to the buffer.
+func (p *printer) printf(format string, args ...interface{}) {
+	fmt.Fprintf(p, format, args...)
+}
+
+// indent returns the position on the current line, in bytes, 0-indexed.
+func (p *printer) indent() int {
+	b := p.Bytes()
+	n := 0
+	for n < len(b) && b[len(b)-1-n] != '\n' {
+		n++
+	}
+	return n
+}
+
+// newline ends the current line, flushing end-of-line comments.
+func (p *printer) newline() {
+	if len(p.comment) > 0 {
+		p.printf(" ")
+		for i, com := range p.comment {
+			if i > 0 {
+				p.trim()
+				p.printf("\n")
+				for i := 0; i < p.margin; i++ {
+					p.printf("\t")
+				}
+			}
+			p.printf("%s", strings.TrimSpace(com.Token))
+		}
+		p.comment = p.comment[:0]
+	}
+
+	p.trim()
+	p.printf("\n")
+	for i := 0; i < p.margin; i++ {
+		p.printf("\t")
+	}
+}
+
+// trim removes trailing spaces and tabs from the current line.
+func (p *printer) trim() {
+	// Remove trailing spaces and tabs from line we're about to end.
+	b := p.Bytes()
+	n := len(b)
+	for n > 0 && (b[n-1] == '\t' || b[n-1] == ' ') {
+		n--
+	}
+	p.Truncate(n)
+}
+
+// file formats the given file into the print buffer.
+func (p *printer) file(f *FileSyntax) {
+	for _, com := range f.Before {
+		p.printf("%s", strings.TrimSpace(com.Token))
+		p.newline()
+	}
+
+	for i, stmt := range f.Stmt {
+		switch x := stmt.(type) {
+		case *CommentBlock:
+			// comments already handled
+			p.expr(x)
+
+		default:
+			p.expr(x)
+			p.newline()
+		}
+
+		for _, com := range stmt.Comment().After {
+			p.printf("%s", strings.TrimSpace(com.Token))
+			p.newline()
+		}
+
+		if i+1 < len(f.Stmt) {
+			p.newline()
+		}
+	}
+}
+
+func (p *printer) expr(x Expr) {
+	// Emit line-comments preceding this expression.
+	if before := x.Comment().Before; len(before) > 0 {
+		// Want to print a line comment.
+		// Line comments must be at the current margin.
+		p.trim()
+		if p.indent() > 0 {
+			// There's other text on the line. Start a new line.
+			p.printf("\n")
+		}
+		// Re-indent to margin.
+		for i := 0; i < p.margin; i++ {
+			p.printf("\t")
+		}
+		for _, com := range before {
+			p.printf("%s", strings.TrimSpace(com.Token))
+			p.newline()
+		}
+	}
+
+	switch x := x.(type) {
+	default:
+		panic(fmt.Errorf("printer: unexpected type %T", x))
+
+	case *CommentBlock:
+		// done
+
+	case *LParen:
+		p.printf("(")
+	case *RParen:
+		p.printf(")")
+
+	case *Line:
+		sep := ""
+		for _, tok := range x.Token {
+			p.printf("%s%s", sep, tok)
+			sep = " "
+		}
+
+	case *LineBlock:
+		for _, tok := range x.Token {
+			p.printf("%s ", tok)
+		}
+		p.expr(&x.LParen)
+		p.margin++
+		for _, l := range x.Line {
+			p.newline()
+			p.expr(l)
+		}
+		p.margin--
+		p.newline()
+		p.expr(&x.RParen)
+	}
+
+	// Queue end-of-line comments for printing when we
+	// reach the end of the line.
+	p.comment = append(p.comment, x.Comment().Suffix...)
+}
diff --git a/internal/confyg/read.go b/internal/confyg/read.go
new file mode 100644
index 0000000..027c4f3
--- /dev/null
+++ b/internal/confyg/read.go
@@ -0,0 +1,680 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Module file parser.
+// This is a simplified copy of Google's buildifier parser.
+
+package confyg
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// A Position describes the position between two bytes of input.
+type Position struct {
+	Line     int // line in input (starting at 1)
+	LineRune int // rune in line (starting at 1)
+	Byte     int // byte in input (starting at 0)
+}
+
+// add returns the position at the end of s, assuming it starts at p.
+func (p Position) add(s string) Position {
+	p.Byte += len(s)
+	if n := strings.Count(s, "\n"); n > 0 {
+		p.Line += n
+		s = s[strings.LastIndex(s, "\n")+1:]
+		p.LineRune = 1
+	}
+	p.LineRune += utf8.RuneCountInString(s)
+	return p
+}
+
+// An Expr represents an input element.
+type Expr interface {
+	// Span returns the start and end position of the expression,
+	// excluding leading or trailing comments.
+	Span() (start, end Position)
+
+	// Comment returns the comments attached to the expression.
+	// This method would normally be named 'Comments' but that
+	// would interfere with embedding a type of the same name.
+	Comment() *Comments
+}
+
+// A Comment represents a single // comment.
+type Comment struct {
+	Start  Position
+	Token  string // without trailing newline
+	Suffix bool   // an end of line (not whole line) comment
+}
+
+// Comments collects the comments associated with an expression.
+type Comments struct {
+	Before []Comment // whole-line comments before this expression
+	Suffix []Comment // end-of-line comments after this expression
+
+	// For top-level expressions only, After lists whole-line
+	// comments following the expression.
+	After []Comment
+}
+
+// Comment returns the receiver. This isn't useful by itself, but
+// a Comments struct is embedded into all the expression
+// implementation types, and this gives each of those a Comment
+// method to satisfy the Expr interface.
+func (c *Comments) Comment() *Comments {
+	return c
+}
+
+// A FileSyntax represents an entire go.mod file.
+type FileSyntax struct {
+	Name string // file path
+	Comments
+	Stmt []Expr
+}
+
+func (x *FileSyntax) Span() (start, end Position) {
+	if len(x.Stmt) == 0 {
+		return
+	}
+	start, _ = x.Stmt[0].Span()
+	_, end = x.Stmt[len(x.Stmt)-1].Span()
+	return start, end
+}
+
+// A CommentBlock represents a top-level block of comments separate
+// from any rule.
+type CommentBlock struct {
+	Comments
+	Start Position
+}
+
+func (x *CommentBlock) Span() (start, end Position) {
+	return x.Start, x.Start
+}
+
+// A Line is a single line of tokens.
+type Line struct {
+	Comments
+	Start Position
+	Token []string
+	End   Position
+}
+
+func (x *Line) Span() (start, end Position) {
+	return x.Start, x.End
+}
+
+// A LineBlock is a factored block of lines, like
+//
+//	require (
+//		"x"
+//		"y"
+//	)
+//
+type LineBlock struct {
+	Comments
+	Start  Position
+	LParen LParen
+	Token  []string
+	Line   []*Line
+	RParen RParen
+}
+
+func (x *LineBlock) Span() (start, end Position) {
+	return x.Start, x.RParen.Pos.add(")")
+}
+
+// An LParen represents the beginning of a parenthesized line block.
+// It is a place to store suffix comments.
+type LParen struct {
+	Comments
+	Pos Position
+}
+
+func (x *LParen) Span() (start, end Position) {
+	return x.Pos, x.Pos.add(")")
+}
+
+// An RParen represents the end of a parenthesized line block.
+// It is a place to store whole-line (before) comments.
+type RParen struct {
+	Comments
+	Pos Position
+}
+
+func (x *RParen) Span() (start, end Position) {
+	return x.Pos, x.Pos.add(")")
+}
+
+// An input represents a single input file being parsed.
+type input struct {
+	// Lexing state.
+	filename  string    // name of input file, for errors
+	complete  []byte    // entire input
+	remaining []byte    // remaining input
+	token     []byte    // token being scanned
+	lastToken string    // most recently returned token, for error messages
+	pos       Position  // current input position
+	comments  []Comment // accumulated comments
+	endRule   int       // position of end of current rule
+
+	// Parser state.
+	file       *FileSyntax // returned top-level syntax tree
+	parseError error       // error encountered during parsing
+
+	// Comment assignment state.
+	pre  []Expr // all expressions, in preorder traversal
+	post []Expr // all expressions, in postorder traversal
+}
+
+func newInput(filename string, data []byte) *input {
+	return &input{
+		filename:  filename,
+		complete:  data,
+		remaining: data,
+		pos:       Position{Line: 1, LineRune: 1, Byte: 0},
+	}
+}
+
+// parse parses the input file.
+func parse(file string, data []byte) (f *FileSyntax, err error) {
+	in := newInput(file, data)
+	// The parser panics for both routine errors like syntax errors
+	// and for programmer bugs like array index errors.
+	// Turn both into error returns. Catching bug panics is
+	// especially important when processing many files.
+	defer func() {
+		if e := recover(); e != nil {
+			if e == in.parseError {
+				err = in.parseError
+			} else {
+				err = fmt.Errorf("%s:%d:%d: internal error: %v", in.filename, in.pos.Line, in.pos.LineRune, e)
+			}
+		}
+	}()
+
+	// Invoke the parser.
+	in.parseFile()
+	if in.parseError != nil {
+		return nil, in.parseError
+	}
+	in.file.Name = in.filename
+
+	// Assign comments to nearby syntax.
+	in.assignComments()
+
+	return in.file, nil
+}
+
+// Error is called to report an error.
+// The reason s is often "syntax error".
+// Error does not return: it panics.
+func (in *input) Error(s string) {
+	if s == "syntax error" && in.lastToken != "" {
+		s += " near " + in.lastToken
+	}
+	in.parseError = fmt.Errorf("%s:%d:%d: %v", in.filename, in.pos.Line, in.pos.LineRune, s)
+	panic(in.parseError)
+}
+
+// eof reports whether the input has reached end of file.
+func (in *input) eof() bool {
+	return len(in.remaining) == 0
+}
+
+// peekRune returns the next rune in the input without consuming it.
+func (in *input) peekRune() int {
+	if len(in.remaining) == 0 {
+		return 0
+	}
+	r, _ := utf8.DecodeRune(in.remaining)
+	return int(r)
+}
+
+// readRune consumes and returns the next rune in the input.
+func (in *input) readRune() int {
+	if len(in.remaining) == 0 {
+		in.Error("internal lexer error: readRune at EOF")
+	}
+	r, size := utf8.DecodeRune(in.remaining)
+	in.remaining = in.remaining[size:]
+	if r == '\n' {
+		in.pos.Line++
+		in.pos.LineRune = 1
+	} else {
+		in.pos.LineRune++
+	}
+	in.pos.Byte += size
+	return int(r)
+}
+
+type symType struct {
+	pos    Position
+	endPos Position
+	text   string
+}
+
+// startToken marks the beginning of the next input token.
+// It must be followed by a call to endToken, once the token has
+// been consumed using readRune.
+func (in *input) startToken(sym *symType) {
+	in.token = in.remaining
+	sym.text = ""
+	sym.pos = in.pos
+}
+
+// endToken marks the end of an input token.
+// It records the actual token string in sym.text if the caller
+// has not done that already.
+func (in *input) endToken(sym *symType) {
+	if sym.text == "" {
+		tok := string(in.token[:len(in.token)-len(in.remaining)])
+		sym.text = tok
+		in.lastToken = sym.text
+	}
+	sym.endPos = in.pos
+}
+
+// lex is called from the parser to obtain the next input token.
+// It returns the token value (either a rune like '+' or a symbolic token _FOR)
+// and sets val to the data associated with the token.
+// For all our input tokens, the associated data is
+// val.Pos (the position where the token begins)
+// and val.Token (the input string corresponding to the token).
+func (in *input) lex(sym *symType) int {
+	// Skip past spaces, stopping at non-space or EOF.
+	countNL := 0 // number of newlines we've skipped past
+	for !in.eof() {
+		// Skip over spaces. Count newlines so we can give the parser
+		// information about where top-level blank lines are,
+		// for top-level comment assignment.
+		c := in.peekRune()
+		if c == ' ' || c == '\t' || c == '\r' {
+			in.readRune()
+			continue
+		}
+
+		// Comment runs to end of line.
+		if c == '#' {
+			in.startToken(sym)
+
+			// Is this comment the only thing on its line?
+			// Find the last \n before this // and see if it's all
+			// spaces from there to here.
+			i := bytes.LastIndex(in.complete[:in.pos.Byte], []byte("\n"))
+			suffix := len(bytes.TrimSpace(in.complete[i+1:in.pos.Byte])) > 0
+			in.readRune()
+			c = in.peekRune()
+			if c != '#' {
+				in.Error(fmt.Sprintf("unexpected input character %#q", c))
+			}
+
+			// Consume comment.
+			for len(in.remaining) > 0 && in.readRune() != '\n' {
+			}
+			in.endToken(sym)
+
+			sym.text = strings.TrimRight(sym.text, "\n")
+			in.lastToken = "comment"
+
+			// If we are at top level (not in a statement), hand the comment to
+			// the parser as a _COMMENT token. The grammar is written
+			// to handle top-level comments itself.
+			if !suffix {
+				// Not in a statement. Tell parser about top-level comment.
+				return _COMMENT
+			}
+
+			// Otherwise, save comment for later attachment to syntax tree.
+			if countNL > 1 {
+				in.comments = append(in.comments, Comment{sym.pos, "", false})
+			}
+			in.comments = append(in.comments, Comment{sym.pos, sym.text, suffix})
+			countNL = 1
+			return _EOL
+		}
+
+		// Found non-space non-comment.
+		break
+	}
+
+	// Found the beginning of the next token.
+	in.startToken(sym)
+	defer in.endToken(sym)
+
+	// End of file.
+	if in.eof() {
+		in.lastToken = "EOF"
+		return _EOF
+	}
+
+	// Punctuation tokens.
+	switch c := in.peekRune(); c {
+	case '\n':
+		in.readRune()
+		return c
+
+	case '(':
+		in.readRune()
+		return c
+
+	case ')':
+		in.readRune()
+		return c
+
+	case '"', '`': // quoted string
+		quote := c
+		in.readRune()
+		for {
+			if in.eof() {
+				in.pos = sym.pos
+				in.Error("unexpected EOF in string")
+			}
+			if in.peekRune() == '\n' {
+				in.Error("unexpected newline in string")
+			}
+			c := in.readRune()
+			if c == quote {
+				break
+			}