update

.gitignore

@@ -1,3 +1,4 @@
 export.json
 .dev.env
-dev
+dev
+*.env

gojq-extended/.gitignore

@@ -0,0 +1 @@
+gojq-extended

gojq-extended/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 chaos
+Copyright (c) 2019-2024 itchyny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gojq-extended/LICENSE.orig

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019-2024 itchyny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gojq-extended/cli.go

@@ -0,0 +1,474 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/mattn/go-isatty"
+
+	"github.com/itchyny/gojq"
+)
+
+const name = "gojq"
+
+const version = "0.12.16"
+
+var revision = "HEAD"
+
+const (
+	exitCodeOK = iota
+	exitCodeFalsyErr
+	exitCodeFlagParseErr
+	exitCodeCompileErr
+	exitCodeNoValueErr
+	exitCodeDefaultErr
+)
+
+type cli struct {
+	inStream  io.Reader
+	outStream io.Writer
+	errStream io.Writer
+
+	outputRaw     bool
+	outputRaw0    bool
+	outputJoin    bool
+	outputCompact bool
+	outputIndent  *int
+	outputTab     bool
+	inputRaw      bool
+	inputStream   bool
+	inputSlurp    bool
+
+	argnames  []string
+	argvalues []any
+
+	exitCodeError error
+}
+
+type flagopts struct {
+	OutputRaw     bool              `short:"r" long:"raw-output" description:"output raw strings"`
+	OutputRaw0    bool              `long:"raw-output0" description:"implies -r with NUL character delimiter"`
+	OutputJoin    bool              `short:"j" long:"join-output" description:"implies -r with no newline delimiter"`
+	OutputCompact bool              `short:"c" long:"compact-output" description:"output without pretty-printing"`
+	OutputIndent  *int              `long:"indent" description:"number of spaces for indentation"`
+	OutputTab     bool              `long:"tab" description:"use tabs for indentation"`
+	OutputColor   bool              `short:"C" long:"color-output" description:"output with colors even if piped"`
+	OutputMono    bool              `short:"M" long:"monochrome-output" description:"output without colors"`
+	InputNull     bool              `short:"n" long:"null-input" description:"use null as input value"`
+	InputRaw      bool              `short:"R" long:"raw-input" description:"read input as raw strings"`
+	InputStream   bool              `long:"stream" description:"parse input in stream fashion"`
+	InputSlurp    bool              `short:"s" long:"slurp" description:"read all inputs into an array"`
+	FromFile      bool              `short:"f" long:"from-file" description:"load query from file"`
+	ModulePaths   []string          `short:"L" description:"directory to search modules from"`
+	Arg           map[string]string `long:"arg" description:"set a string value to a variable"`
+	ArgJSON       map[string]string `long:"argjson" description:"set a JSON value to a variable"`
+	SlurpFile     map[string]string `long:"slurpfile" description:"set the JSON contents of a file to a variable"`
+	RawFile       map[string]string `long:"rawfile" description:"set the contents of a file to a variable"`
+	Args          []any             `long:"args" positional:"" description:"consume remaining arguments as positional string values"`
+	JSONArgs      []any             `long:"jsonargs" positional:"" description:"consume remaining arguments as positional JSON values"`
+	ExitStatus    bool              `short:"e" long:"exit-status" description:"exit 1 when the last value is false or null"`
+	Version       bool              `short:"v" long:"version" description:"display version information"`
+	Help          bool              `short:"h" long:"help" description:"display this help information"`
+}
+
+var addDefaultModulePaths = true
+
+func (cli *cli) run(args []string) int {
+	if err := cli.runInternal(args); err != nil {
+		if _, ok := err.(interface{ isEmptyError() }); !ok {
+			fmt.Fprintf(cli.errStream, "%s: %s\n", name, err)
+		}
+		if err, ok := err.(interface{ ExitCode() int }); ok {
+			return err.ExitCode()
+		}
+		return exitCodeDefaultErr
+	}
+	return exitCodeOK
+}
+
+func (cli *cli) runInternal(args []string) (err error) {
+	var opts flagopts
+	args, err = parseFlags(args, &opts)
+	if err != nil {
+		return &flagParseError{err}
+	}
+	if opts.Help {
+		fmt.Fprintf(cli.outStream, `%[1]s - Go implementation of jq
+
+Version: %s (rev: %s/%s)
+
+Synopsis:
+  %% echo '{"foo": 128}' | %[1]s '.foo'
+
+Usage:
+  %[1]s [OPTIONS]
+
+`,
+			name, version, revision, runtime.Version())
+		fmt.Fprintln(cli.outStream, formatFlags(&opts))
+		return nil
+	}
+	if opts.Version {
+		fmt.Fprintf(cli.outStream, "%s %s (rev: %s/%s)\n", name, version, revision, runtime.Version())
+		return nil
+	}
+	cli.outputRaw, cli.outputRaw0, cli.outputJoin,
+		cli.outputCompact, cli.outputIndent, cli.outputTab =
+		opts.OutputRaw, opts.OutputRaw0, opts.OutputJoin,
+		opts.OutputCompact, opts.OutputIndent, opts.OutputTab
+	defer func(x bool) { noColor = x }(noColor)
+	if opts.OutputColor || opts.OutputMono {
+		noColor = opts.OutputMono
+	} else if os.Getenv("NO_COLOR") != "" || os.Getenv("TERM") == "dumb" {
+		noColor = true
+	} else {
+		f, ok := cli.outStream.(interface{ Fd() uintptr })
+		noColor = !(ok && (isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd())))
+	}
+	if !noColor {
+		if colors := os.Getenv("GOJQ_COLORS"); colors != "" {
+			if err := setColors(colors); err != nil {
+				return err
+			}
+		}
+	}
+	if i := cli.outputIndent; i != nil {
+		if *i > 9 {
+			return fmt.Errorf("too many indentation count: %d", *i)
+		} else if *i < 0 {
+			return fmt.Errorf("negative indentation count: %d", *i)
+		}
+	}
+
+	cli.inputRaw, cli.inputStream, cli.inputSlurp =
+		opts.InputRaw, opts.InputStream, opts.InputSlurp
+	for k, v := range opts.Arg {
+		cli.argnames = append(cli.argnames, "$"+k)
+		cli.argvalues = append(cli.argvalues, v)
+	}
+	for k, v := range opts.ArgJSON {
+		val, _ := newJSONInputIter(strings.NewReader(v), "$"+k).Next()
+		if err, ok := val.(error); ok {
+			return err
+		}
+		cli.argnames = append(cli.argnames, "$"+k)
+		cli.argvalues = append(cli.argvalues, val)
+	}
+	for k, v := range opts.SlurpFile {
+		val, err := slurpFile(v)
+		if err != nil {
+			return err
+		}
+		cli.argnames = append(cli.argnames, "$"+k)
+		cli.argvalues = append(cli.argvalues, val)
+	}
+	for k, v := range opts.RawFile {
+		val, err := os.ReadFile(v)
+		if err != nil {
+			return err
+		}
+		cli.argnames = append(cli.argnames, "$"+k)
+		cli.argvalues = append(cli.argvalues, string(val))
+	}
+	named := make(map[string]any, len(cli.argnames))
+	for i, name := range cli.argnames {
+		named[name[1:]] = cli.argvalues[i]
+	}
+	positional := opts.Args
+	for i, v := range opts.JSONArgs {
+		if v != nil {
+			val, _ := newJSONInputIter(strings.NewReader(v.(string)), "--jsonargs").Next()
+			if err, ok := val.(error); ok {
+				return err
+			}
+			if i < len(positional) {
+				positional[i] = val
+			} else {
+				positional = append(positional, val)
+			}
+		}
+	}
+	cli.argnames = append(cli.argnames, "$ARGS")
+	cli.argvalues = append(cli.argvalues, map[string]any{
+		"named":      named,
+		"positional": positional,
+	})
+	var arg, fname string
+	if opts.FromFile {
+		if len(args) == 0 {
+			return errors.New("expected a query file for flag `-f'")
+		}
+		src, err := os.ReadFile(args[0])
+		if err != nil {
+			return err
+		}
+		arg, args, fname = string(src), args[1:], args[0]
+	} else if len(args) == 0 {
+		arg = "."
+	} else {
+		arg, args, fname = strings.TrimSpace(args[0]), args[1:], "<arg>"
+	}
+	if opts.ExitStatus {
+		cli.exitCodeError = &exitCodeError{exitCodeNoValueErr}
+		defer func() {
+			if _, ok := err.(interface{ ExitCode() int }); !ok {
+				err = cli.exitCodeError
+			}
+		}()
+	}
+	query, err := gojq.Parse(arg)
+	if err != nil {
+		return &queryParseError{fname, arg, err}
+	}
+	modulePaths := opts.ModulePaths
+	if len(modulePaths) == 0 && addDefaultModulePaths {
+		modulePaths = []string{"~/.jq", "$ORIGIN/../lib/gojq", "$ORIGIN/../lib"}
+	}
+	iter := cli.createInputIter(args)
+	defer iter.Close()
+	code, err := gojq.Compile(query,
+		gojq.WithModuleLoader(gojq.NewModuleLoader(modulePaths)),
+		gojq.WithEnvironLoader(os.Environ),
+		gojq.WithVariables(cli.argnames),
+		gojq.WithFunction("debug", 0, 0,
+			func(errStream io.Writer) func(any, []any) any {
+				indent := 2
+				if cli.outputCompact {
+					indent = 0
+				} else if cli.outputTab {
+					indent = 1
+				} else if i := cli.outputIndent; i != nil {
+					indent = *i
+				}
+
+				return func(v any, _ []any) any {
+					if err := newEncoder(false, indent).
+						marshal([]any{"DEBUG:", v}, cli.errStream); err != nil {
+						return err
+					}
+					if _, err := cli.errStream.Write([]byte{'\n'}); err != nil {
+						return err
+					}
+					return v
+				}
+			}(cli.errStream),
+		),
+		gojq.WithFunction("stderr", 0, 0,
+			func(errStream io.Writer) func(any, []any) any {
+				return func(v any, _ []any) any {
+					if err := (&rawMarshaler{m: newEncoder(false, 0)}).
+						marshal(v, cli.errStream); err != nil {
+						return err
+					}
+					return v
+				}
+			}(cli.errStream),
+		),
+		gojq.WithFunction("_readFile", 1, 1,
+			func(_input any, args []any) any {
+				filename := args[0].(string)
+
+				info, err := os.Stat(filename)
+				if err != nil {
+					return fmt.Errorf("file not found")
+				}
+
+				if info.IsDir() {
+					return fmt.Errorf("file is a directory")
+				}
+
+				content, err := os.ReadFile(filename)
+				if err != nil {
+					return fmt.Errorf("file could not be read")
+				}
+				return string(content)
+			},
+		),
+		gojq.WithFunction("_writeFileString", 1, 1,
+			func(input any, args []any) any {
+				filename := args[0].(string)
+				content := ""
+
+				switch input := input.(type) {
+				case string:
+					content = input
+				default:
+					return fmt.Errorf("invalid type passed to _writeFileString")
+				}
+
+				file, err := os.Create(filename)
+				if err != nil {
+					return err
+				}
+
+				_, err = file.Write([]byte(content))
+				if err != nil {
+					return err
+				}
+				err = file.Close()
+				if err != nil {
+					return err
+				}
+
+				return nil
+			},
+		),
+		gojq.WithFunction("input_filename", 0, 0,
+			func(iter inputIter) func(any, []any) any {
+				return func(any, []any) any {
+					if fname := iter.Name(); fname != "" && (len(args) > 0 || !opts.InputNull) {
+						return fname
+					}
+					return nil
+				}
+			}(iter),
+		),
+		gojq.WithInputIter(iter),
+	)
+	if err != nil {
+		if err, ok := err.(interface {
+			QueryParseError() (string, string, error)
+		}); ok {
+			name, query, err := err.QueryParseError()
+			return &queryParseError{name, query, err}
+		}
+		if err, ok := err.(interface {
+			JSONParseError() (string, string, error)
+		}); ok {
+			fname, contents, err := err.JSONParseError()
+			return &compileError{&jsonParseError{fname, contents, 0, err}}
+		}
+		return &compileError{err}
+	}
+	if opts.InputNull {
+		iter = newNullInputIter()
+	}
+	return cli.process(iter, code)
+}
+
+func slurpFile(name string) (any, error) {
+	iter := newSlurpInputIter(
+		newFilesInputIter(newJSONInputIter, []string{name}, nil),
+	)
+	defer iter.Close()
+	val, _ := iter.Next()
+	if err, ok := val.(error); ok {
+		return nil, err
+	}
+	return val, nil
+}
+
+func (cli *cli) createInputIter(args []string) (iter inputIter) {
+	var newIter func(io.Reader, string) inputIter
+	switch {
+	case cli.inputRaw:
+		if cli.inputSlurp {
+			newIter = newReadAllIter
+		} else {
+			newIter = newRawInputIter
+		}
+	case cli.inputStream:
+		newIter = newStreamInputIter
+	default:
+		newIter = newJSONInputIter
+	}
+	if cli.inputSlurp {
+		defer func() {
+			if cli.inputRaw {
+				iter = newSlurpRawInputIter(iter)
+			} else {
+				iter = newSlurpInputIter(iter)
+			}
+		}()
+	}
+	if len(args) == 0 {
+		return newIter(cli.inStream, "<stdin>")
+	}
+	return newFilesInputIter(newIter, args, cli.inStream)
+}
+
+func (cli *cli) process(iter inputIter, code *gojq.Code) error {
+	var err error
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if e, ok := v.(error); ok {
+			fmt.Fprintf(cli.errStream, "%s: %s\n", name, e)
+			err = e
+			continue
+		}
+		if e := cli.printValues(code.Run(v, cli.argvalues...)); e != nil {
+			if e, ok := e.(*gojq.HaltError); ok {
+				if v := e.Value(); v != nil {
+					if str, ok := v.(string); ok {
+						cli.errStream.Write([]byte(str))
+					} else {
+						bs, _ := gojq.Marshal(v)
+						cli.errStream.Write(bs)
+						cli.errStream.Write([]byte{'\n'})
+					}
+				}
+				err = e
+				break
+			}
+			fmt.Fprintf(cli.errStream, "%s: %s\n", name, e)
+			err = e
+		}
+	}
+	if err != nil {
+		return &emptyError{err}
+	}
+	return nil
+}
+
+func (cli *cli) printValues(iter gojq.Iter) error {
+	m := cli.createMarshaler()
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			return err
+		}
+
+		if err := m.marshal(v, cli.outStream); err != nil {
+			return err
+		}
+		if cli.exitCodeError != nil {
+			if v == nil || v == false {
+				cli.exitCodeError = &exitCodeError{exitCodeFalsyErr}
+			} else {
+				cli.exitCodeError = &exitCodeError{exitCodeOK}
+			}
+		}
+		if cli.outputRaw0 {
+			cli.outStream.Write([]byte{'\x00'})
+		} else if !cli.outputJoin {
+			cli.outStream.Write([]byte{'\n'})
+		}
+	}
+	return nil
+}
+
+func (cli *cli) createMarshaler() marshaler {
+	indent := 2
+	if cli.outputCompact {
+		indent = 0
+	} else if cli.outputTab {
+		indent = 1
+	} else if i := cli.outputIndent; i != nil {
+		indent = *i
+	}
+	f := newEncoder(cli.outputTab, indent)
+	if cli.outputRaw || cli.outputRaw0 || cli.outputJoin {
+		return &rawMarshaler{f, cli.outputRaw0}
+	}
+	return f
+}

gojq-extended/color.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+var noColor bool
+
+func newColor(c string) []byte {
+	return []byte("\x1b[" + c + "m")
+}
+
+func setColor(buf *bytes.Buffer, color []byte) {
+	if !noColor {
+		buf.Write(color)
+	}
+}
+
+var (
+	resetColor     = newColor("0")    // Reset
+	nullColor      = newColor("90")   // Bright black
+	falseColor     = newColor("33")   // Yellow
+	trueColor      = newColor("33")   // Yellow
+	numberColor    = newColor("36")   // Cyan
+	stringColor    = newColor("32")   // Green
+	objectKeyColor = newColor("34;1") // Bold Blue
+	arrayColor     = []byte(nil)      // No color
+	objectColor    = []byte(nil)      // No color
+)
+
+func validColor(x string) bool {
+	var num bool
+	for _, c := range x {
+		if '0' <= c && c <= '9' {
+			num = true
+		} else if c == ';' && num {
+			num = false
+		} else {
+			return false
+		}
+	}
+	return num
+}
+
+func setColors(colors string) error {
+	var color string
+	for _, target := range []*[]byte{
+		&nullColor, &falseColor, &trueColor, &numberColor,
+		&stringColor, &objectKeyColor, &arrayColor, &objectColor,
+	} {
+		color, colors, _ = strings.Cut(colors, ":")
+		if color != "" {
+			if !validColor(color) {
+				return fmt.Errorf("invalid color: %q", color)
+			}
+			*target = newColor(color)
+		} else {
+			*target = nil
+		}
+	}
+	return nil
+}

gojq-extended/encoder.go

@@ -0,0 +1,267 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"math/big"
+	"sort"
+	"strconv"
+	"unicode/utf8"
+)
+
+type encoder struct {
+	out    io.Writer
+	w      *bytes.Buffer
+	tab    bool
+	indent int
+	depth  int
+	buf    [64]byte
+}
+
+func newEncoder(tab bool, indent int) *encoder {
+	// reuse the buffer in multiple calls of marshal
+	return &encoder{w: new(bytes.Buffer), tab: tab, indent: indent}
+}
+
+func (e *encoder) flush() error {
+	_, err := e.out.Write(e.w.Bytes())
+	e.w.Reset()
+	return err
+}
+
+func (e *encoder) marshal(v any, w io.Writer) error {
+	e.out = w
+	err := e.encode(v)
+	if ferr := e.flush(); ferr != nil && err == nil {
+		err = ferr
+	}
+	return err
+}
+
+func (e *encoder) encode(v any) error {
+	switch v := v.(type) {
+	case nil:
+		e.write([]byte("null"), nullColor)
+	case bool:
+		if v {
+			e.write([]byte("true"), trueColor)
+		} else {
+			e.write([]byte("false"), falseColor)
+		}
+	case int:
+		e.write(strconv.AppendInt(e.buf[:0], int64(v), 10), numberColor)
+	case float64:
+		e.encodeFloat64(v)
+	case *big.Int:
+		e.write(v.Append(e.buf[:0], 10), numberColor)
+	case string:
+		e.encodeString(v, stringColor)
+	case []any:
+		if err := e.encodeArray(v); err != nil {
+			return err
+		}
+	case map[string]any:
+		if err := e.encodeObject(v); err != nil {
+			return err
+		}
+	default:
+		panic(fmt.Sprintf("invalid type: %[1]T (%[1]v)", v))
+	}
+	if e.w.Len() > 8*1024 {
+		return e.flush()
+	}
+	return nil
+}
+
+// ref: floatEncoder in encoding/json
+func (e *encoder) encodeFloat64(f float64) {
+	if math.IsNaN(f) {
+		e.write([]byte("null"), nullColor)
+		return
+	}
+	f = min(max(f, -math.MaxFloat64), math.MaxFloat64)
+	format := byte('f')
+	if x := math.Abs(f); x != 0 && x < 1e-6 || x >= 1e21 {
+		format = 'e'
+	}
+	buf := strconv.AppendFloat(e.buf[:0], f, format, -1, 64)
+	if format == 'e' {
+		// clean up e-09 to e-9
+		if n := len(buf); n >= 4 && buf[n-4] == 'e' && buf[n-3] == '-' && buf[n-2] == '0' {
+			buf[n-2] = buf[n-1]
+			buf = buf[:n-1]
+		}
+	}
+	e.write(buf, numberColor)
+}
+
+// ref: encodeState#string in encoding/json
+func (e *encoder) encodeString(s string, color []byte) {
+	if color != nil {
+		setColor(e.w, color)
+	}
+	e.w.WriteByte('"')
+	start := 0
+	for i := 0; i < len(s); {
+		if b := s[i]; b < utf8.RuneSelf {
+			if ' ' <= b && b <= '~' && b != '"' && b != '\\' {
+				i++
+				continue
+			}
+			if start < i {
+				e.w.WriteString(s[start:i])
+			}
+			switch b {
+			case '"':
+				e.w.WriteString(`\"`)
+			case '\\':
+				e.w.WriteString(`\\`)
+			case '\b':
+				e.w.WriteString(`\b`)
+			case '\f':
+				e.w.WriteString(`\f`)
+			case '\n':
+				e.w.WriteString(`\n`)
+			case '\r':
+				e.w.WriteString(`\r`)
+			case '\t':
+				e.w.WriteString(`\t`)
+			default:
+				const hex = "0123456789abcdef"
+				e.w.WriteString(`\u00`)
+				e.w.WriteByte(hex[b>>4])
+				e.w.WriteByte(hex[b&0xF])
+			}
+			i++
+			start = i
+			continue
+		}
+		c, size := utf8.DecodeRuneInString(s[i:])
+		if c == utf8.RuneError && size == 1 {
+			if start < i {
+				e.w.WriteString(s[start:i])
+			}
+			e.w.WriteString(`\ufffd`)
+			i += size
+			start = i
+			continue
+		}
+		i += size
+	}
+	if start < len(s) {
+		e.w.WriteString(s[start:])
+	}
+	e.w.WriteByte('"')
+	if color != nil {
+		setColor(e.w, resetColor)
+	}
+}
+
+func (e *encoder) encodeArray(vs []any) error {
+	e.writeByte('[', arrayColor)
+	e.depth += e.indent
+	for i, v := range vs {
+		if i > 0 {
+			e.writeByte(',', arrayColor)
+		}
+		if e.indent != 0 {
+			e.writeIndent()
+		}
+		if err := e.encode(v); err != nil {
+			return err
+		}
+	}
+	e.depth -= e.indent
+	if len(vs) > 0 && e.indent != 0 {
+		e.writeIndent()
+	}
+	e.writeByte(']', arrayColor)
+	return nil
+}
+
+func (e *encoder) encodeObject(vs map[string]any) error {
+	e.writeByte('{', objectColor)
+	e.depth += e.indent
+	type keyVal struct {
+		key string
+		val any
+	}
+	kvs := make([]keyVal, len(vs))
+	var i int
+	for k, v := range vs {
+		kvs[i] = keyVal{k, v}
+		i++
+	}
+	sort.Slice(kvs, func(i, j int) bool {
+		return kvs[i].key < kvs[j].key
+	})
+	for i, kv := range kvs {
+		if i > 0 {
+			e.writeByte(',', objectColor)
+		}
+		if e.indent != 0 {
+			e.writeIndent()
+		}
+		e.encodeString(kv.key, objectKeyColor)
+		e.writeByte(':', objectColor)
+		if e.indent != 0 {
+			e.w.WriteByte(' ')
+		}
+		if err := e.encode(kv.val); err != nil {
+			return err
+		}
+	}
+	e.depth -= e.indent
+	if len(vs) > 0 && e.indent != 0 {
+		e.writeIndent()
+	}
+	e.writeByte('}', objectColor)
+	return nil
+}
+
+func (e *encoder) writeIndent() {
+	e.w.WriteByte('\n')
+	if n := e.depth; n > 0 {
+		if e.tab {
+			e.writeIndentInternal(n, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t")
+		} else {
+			e.writeIndentInternal(n, "                                ")
+		}
+	}
+}
+
+func (e *encoder) writeIndentInternal(n int, spaces string) {
+	if l := len(spaces); n <= l {
+		e.w.WriteString(spaces[:n])
+	} else {
+		e.w.WriteString(spaces)
+		for n -= l; n > 0; n, l = n-l, l*2 {
+			if n < l {
+				l = n
+			}
+			e.w.Write(e.w.Bytes()[e.w.Len()-l:])
+		}
+	}
+}
+
+func (e *encoder) writeByte(b byte, color []byte) {
+	if color == nil {
+		e.w.WriteByte(b)
+	} else {
+		setColor(e.w, color)
+		e.w.WriteByte(b)
+		setColor(e.w, resetColor)
+	}
+}
+
+func (e *encoder) write(bs []byte, color []byte) {
+	if color == nil {
+		e.w.Write(bs)
+	} else {
+		setColor(e.w, color)
+		e.w.Write(bs)
+		setColor(e.w, resetColor)
+	}
+}

gojq-extended/error.go

@@ -0,0 +1,225 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/mattn/go-runewidth"
+
+	"github.com/itchyny/gojq"
+)
+
+type emptyError struct {
+	err error
+}
+
+func (*emptyError) Error() string {
+	return ""
+}
+
+func (*emptyError) isEmptyError() {}
+
+func (err *emptyError) ExitCode() int {
+	if err, ok := err.err.(interface{ ExitCode() int }); ok {
+		return err.ExitCode()
+	}
+	return exitCodeDefaultErr
+}
+
+type exitCodeError struct {
+	code int
+}
+
+func (err *exitCodeError) Error() string {
+	return "exit code: " + strconv.Itoa(err.code)
+}
+
+func (*exitCodeError) isEmptyError() {}
+
+func (err *exitCodeError) ExitCode() int {
+	return err.code
+}
+
+type flagParseError struct {
+	err error
+}
+
+func (err *flagParseError) Error() string {
+	return err.err.Error()
+}
+
+func (*flagParseError) ExitCode() int {
+	return exitCodeFlagParseErr
+}
+
+type compileError struct {
+	err error
+}
+
+func (err *compileError) Error() string {
+	return "compile error: " + err.err.Error()
+}
+
+func (*compileError) ExitCode() int {
+	return exitCodeCompileErr
+}
+
+type queryParseError struct {
+	fname, contents string
+	err             error
+}
+
+func (err *queryParseError) Error() string {
+	var offset int
+	var e *gojq.ParseError
+	if errors.As(err.err, &e) {
+		offset = e.Offset - len(e.Token) + 1
+	}
+	linestr, line, column := getLineByOffset(err.contents, offset)
+	if err.fname != "<arg>" || containsNewline(err.contents) {
+		return fmt.Sprintf("invalid query: %s:%d\n%s  %s",
+			err.fname, line, formatLineInfo(linestr, line, column), err.err)
+	}
+	return fmt.Sprintf("invalid query: %s\n    %s\n    %*c  %s",
+		err.contents, linestr, column+1, '^', err.err)
+}
+
+func (*queryParseError) ExitCode() int {
+	return exitCodeCompileErr
+}
+
+type jsonParseError struct {
+	fname, contents string
+	line            int
+	err             error
+}
+
+func (err *jsonParseError) Error() string {
+	var offset int
+	if err.err == io.ErrUnexpectedEOF {
+		offset = len(err.contents) + 1
+	} else if e, ok := err.err.(*json.SyntaxError); ok {
+		offset = int(e.Offset)
+	}
+	linestr, line, column := getLineByOffset(err.contents, offset)
+	if line += err.line; line > 1 {
+		return fmt.Sprintf("invalid json: %s:%d\n%s  %s",
+			err.fname, line, formatLineInfo(linestr, line, column), err.err)
+	}
+	return fmt.Sprintf("invalid json: %s\n    %s\n    %*c  %s",
+		err.fname, linestr, column+1, '^', err.err)
+}
+
+func getLineByOffset(str string, offset int) (linestr string, line, column int) {
+	ss := &stringScanner{str, 0}
+	for {
+		str, start, ok := ss.next()
+		if !ok {
+			offset -= start
+			break
+		}
+		line++
+		linestr = str
+		if ss.offset >= offset {
+			offset -= start
+			break
+		}
+	}
+	offset = min(max(offset-1, 0), len(linestr))
+	if offset > 48 {
+		skip := len(trimLastInvalidRune(linestr[:offset-48]))
+		linestr = linestr[skip:]
+		offset -= skip
+	}
+	linestr = trimLastInvalidRune(linestr[:min(64, len(linestr))])
+	if offset < len(linestr) {
+		offset = len(trimLastInvalidRune(linestr[:offset]))
+	} else {
+		offset = len(linestr)
+	}
+	column = runewidth.StringWidth(linestr[:offset])
+	return
+}
+
+func getLineByLine(str string, line int) (linestr string) {
+	ss := &stringScanner{str, 0}
+	for {
+		str, _, ok := ss.next()
+		if !ok {
+			break
+		}
+		if line--; line == 0 {
+			linestr = str
+			break
+		}
+	}
+	if len(linestr) > 64 {
+		linestr = trimLastInvalidRune(linestr[:64])
+	}
+	return
+}
+
+func trimLastInvalidRune(s string) string {
+	for i := len(s) - 1; i >= 0 && i > len(s)-utf8.UTFMax; i-- {
+		if b := s[i]; b < utf8.RuneSelf {
+			return s[:i+1]
+		} else if utf8.RuneStart(b) {
+			if r, _ := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError {
+				return s[:i]
+			}
+			break
+		}
+	}
+	return s
+}
+
+func formatLineInfo(linestr string, line, column int) string {
+	l := strconv.Itoa(line)
+	return fmt.Sprintf("    %s | %s\n    %*c", l, linestr, column+len(l)+4, '^')
+}
+
+type stringScanner struct {
+	str    string
+	offset int
+}
+
+func (ss *stringScanner) next() (line string, start int, ok bool) {
+	if ss.offset == len(ss.str) {
+		return
+	}
+	start, ok = ss.offset, true
+	line = ss.str[start:]
+	i := indexNewline(line)
+	if i < 0 {
+		ss.offset = len(ss.str)
+		return
+	}
+	line = line[:i]
+	if strings.HasPrefix(ss.str[start+i:], "\r\n") {
+		i++
+	}
+	ss.offset += i + 1
+	return
+}
+
+// Faster than strings.ContainsAny(str, "\r\n").
+func containsNewline(str string) bool {
+	return strings.IndexByte(str, '\n') >= 0 ||
+		strings.IndexByte(str, '\r') >= 0
+}
+
+// Faster than strings.IndexAny(str, "\r\n").
+func indexNewline(str string) (i int) {
+	if i = strings.IndexByte(str, '\n'); i >= 0 {
+		str = str[:i]
+	}
+	if j := strings.IndexByte(str, '\r'); j >= 0 {
+		i = j
+	}
+	return
+}

gojq-extended/flags.go

@@ -0,0 +1,211 @@
+package main
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+func parseFlags(args []string, opts any) ([]string, error) {
+	rest := make([]string, 0, len(args))
+	val := reflect.ValueOf(opts).Elem()
+	typ := val.Type()
+	longToValue := map[string]reflect.Value{}
+	longToPositional := map[string]struct{}{}
+	shortToValue := map[string]reflect.Value{}
+	for i, l := 0, val.NumField(); i < l; i++ {
+		if flag, ok := typ.Field(i).Tag.Lookup("long"); ok {
+			longToValue[flag] = val.Field(i)
+			if _, ok := typ.Field(i).Tag.Lookup("positional"); ok {
+				longToPositional[flag] = struct{}{}
+			}
+		}
+		if flag, ok := typ.Field(i).Tag.Lookup("short"); ok {
+			shortToValue[flag] = val.Field(i)
+		}
+	}
+	mapKeys := map[string]struct{}{}
+	var positionalVal reflect.Value
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		var (
+			val       reflect.Value
+			ok        bool
+			shortopts string
+		)
+		if arg == "--" {
+			if positionalVal.IsValid() {
+				for _, arg := range args[i+1:] {
+					positionalVal.Set(reflect.Append(positionalVal, reflect.ValueOf(arg)))
+				}
+			} else {
+				rest = append(rest, args[i+1:]...)
+			}
+			break
+		}
+		if strings.HasPrefix(arg, "--") {
+			if val, ok = longToValue[arg[2:]]; !ok {
+				if j := strings.IndexByte(arg, '='); j >= 0 {
+					if val, ok = longToValue[arg[2:j]]; ok {
+						if val.Kind() == reflect.Bool {
+							return nil, fmt.Errorf("boolean flag `%s' cannot have an argument", arg[:j])
+						}
+						args[i] = arg[j+1:]
+						arg = arg[:j]
+						i--
+					}
+				}
+				if !ok {
+					return nil, fmt.Errorf("unknown flag `%s'", arg)
+				}
+			}
+		} else if len(arg) > 1 && arg[0] == '-' {
+			var skip bool
+			for i := 1; i < len(arg); i++ {
+				opt := arg[i : i+1]
+				if val, ok = shortToValue[opt]; ok {
+					if val.Kind() != reflect.Bool {
+						break
+					}
+				} else if !("A" <= opt && opt <= "Z" || "a" <= opt && opt <= "z") {
+					skip = true
+					break
+				}
+			}
+			if !skip && (len(arg) > 2 || !ok) {
+				shortopts = arg[1:]
+				goto L
+			}
+		}
+		if !ok {
+			if positionalVal.IsValid() && len(rest) > 0 {
+				positionalVal.Set(reflect.Append(positionalVal, reflect.ValueOf(arg)))
+			} else {
+				rest = append(rest, arg)
+			}
+			continue
+		}
+	S:
+		switch val.Kind() {
+		case reflect.Bool:
+			val.SetBool(true)
+		case reflect.String:
+			if i++; i >= len(args) {
+				return nil, fmt.Errorf("expected argument for flag `%s'", arg)
+			}
+			val.SetString(args[i])
+		case reflect.Ptr:
+			if val.Type().Elem().Kind() == reflect.Int {
+				if i++; i >= len(args) {
+					return nil, fmt.Errorf("expected argument for flag `%s'", arg)
+				}
+				v, err := strconv.Atoi(args[i])
+				if err != nil {
+					return nil, fmt.Errorf("invalid argument for flag `%s': %w", arg, err)
+				}
+				val.Set(reflect.New(val.Type().Elem()))
+				val.Elem().SetInt(int64(v))
+			}
+		case reflect.Slice:
+			if _, ok := longToPositional[arg[2:]]; ok {
+				if positionalVal.IsValid() {
+					for positionalVal.Len() > val.Len() {
+						val.Set(reflect.Append(val, reflect.Zero(val.Type().Elem())))
+					}
+				}
+				positionalVal = val
+			} else {
+				if i++; i >= len(args) {
+					return nil, fmt.Errorf("expected argument for flag `%s'", arg)
+				}
+				val.Set(reflect.Append(val, reflect.ValueOf(args[i])))
+			}
+		case reflect.Map:
+			if i += 2; i >= len(args) {
+				return nil, fmt.Errorf("expected 2 arguments for flag `%s'", arg)
+			}
+			if val.IsNil() {
+				val.Set(reflect.MakeMap(val.Type()))
+			}
+			name := args[i-1]
+			if _, ok := mapKeys[name]; !ok {
+				mapKeys[name] = struct{}{}
+				val.SetMapIndex(reflect.ValueOf(name), reflect.ValueOf(args[i]))
+			}
+		}
+	L:
+		if shortopts != "" {
+			opt := shortopts[:1]
+			if val, ok = shortToValue[opt]; !ok {
+				return nil, fmt.Errorf("unknown flag `%s'", opt)
+			}
+			if val.Kind() != reflect.Bool && len(shortopts) > 1 {
+				if shortopts[1] == '=' {
+					args[i] = shortopts[2:]
+				} else {
+					args[i] = shortopts[1:]
+				}
+				i--
+				shortopts = ""
+			} else {
+				shortopts = shortopts[1:]
+			}
+			arg = "-" + opt
+			goto S
+		}
+	}
+	return rest, nil
+}
+
+func formatFlags(opts any) string {
+	val := reflect.ValueOf(opts).Elem()
+	typ := val.Type()
+	var sb strings.Builder
+	sb.WriteString("Command Options:\n")
+	for i, l := 0, typ.NumField(); i < l; i++ {
+		tag := typ.Field(i).Tag
+		if i == l-1 {
+			sb.WriteString("\nHelp Option:\n")
+		}
+		sb.WriteString("  ")
+		var short bool
+		if flag, ok := tag.Lookup("short"); ok {
+			sb.WriteString("-")
+			sb.WriteString(flag)
+			short = true
+		} else {
+			sb.WriteString("  ")
+		}
+		m := sb.Len()
+		if flag, ok := tag.Lookup("long"); ok {
+			if short {
+				sb.WriteString(", ")
+			} else {
+				sb.WriteString("  ")
+			}
+			sb.WriteString("--")
+			sb.WriteString(flag)
+			switch val.Field(i).Kind() {
+			case reflect.Bool:
+				sb.WriteString(" ")
+			case reflect.Map:
+				if strings.HasSuffix(flag, "file") {
+					sb.WriteString(" name file")
+				} else {
+					sb.WriteString(" name value")
+				}
+			default:
+				if _, ok = tag.Lookup("positional"); !ok {
+					sb.WriteString("=")
+				}
+			}
+		} else {
+			sb.WriteString("=")
+		}
+		sb.WriteString("                       "[:24-sb.Len()+m])
+		sb.WriteString(tag.Get("description"))
+		sb.WriteString("\n")
+	}
+	return sb.String()
+}

gojq-extended/go.mod

@@ -0,0 +1,15 @@
+module forgejo.owo.monster/chaos/psychonaut_journal_cli/gojq-extended
+
+go 1.23.2
+
+require (
+	github.com/itchyny/gojq v0.12.16
+	github.com/mattn/go-isatty v0.0.20
+	github.com/mattn/go-runewidth v0.0.16
+)
+
+require (
+	github.com/itchyny/timefmt-go v0.1.6 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)

gojq-extended/go.sum

@@ -0,0 +1,14 @@
+github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
+github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
+github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
+github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

gojq-extended/inputs.go

@@ -0,0 +1,375 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/itchyny/gojq"
+)
+
+type inputReader struct {
+	io.Reader
+	file *os.File
+	buf  *bytes.Buffer
+}
+
+func newInputReader(r io.Reader) *inputReader {
+	if r, ok := r.(*os.File); ok {
+		if _, err := r.Seek(0, io.SeekCurrent); err == nil {
+			return &inputReader{r, r, nil}
+		}
+	}
+	var buf bytes.Buffer // do not use strings.Builder because we need to Reset
+	return &inputReader{io.TeeReader(r, &buf), nil, &buf}
+}
+
+func (ir *inputReader) getContents(offset *int64, line *int) string {
+	if buf := ir.buf; buf != nil {
+		return buf.String()
+	}
+	if current, err := ir.file.Seek(0, io.SeekCurrent); err == nil {
+		defer func() { ir.file.Seek(current, io.SeekStart) }()
+	}
+	ir.file.Seek(0, io.SeekStart)
+	const bufSize = 16 * 1024
+	var buf bytes.Buffer // do not use strings.Builder because we need to Reset
+	if offset != nil && *offset > bufSize {
+		buf.Grow(bufSize)
+		for *offset > bufSize {
+			n, err := io.Copy(&buf, io.LimitReader(ir.file, bufSize))
+			*offset -= int64(n)
+			*line += bytes.Count(buf.Bytes(), []byte{'\n'})
+			buf.Reset()
+			if err != nil || n == 0 {
+				break
+			}
+		}
+	}
+	var r io.Reader
+	if offset == nil {
+		r = ir.file
+	} else {
+		r = io.LimitReader(ir.file, bufSize*2)
+	}
+	io.Copy(&buf, r)
+	return buf.String()
+}
+
+type inputIter interface {
+	gojq.Iter
+	io.Closer
+	Name() string
+}
+
+type jsonInputIter struct {
+	next   func() (any, error)
+	ir     *inputReader
+	fname  string
+	offset int64
+	line   int
+	err    error
+}
+
+func newJSONInputIter(r io.Reader, fname string) inputIter {
+	ir := newInputReader(r)
+	dec := json.NewDecoder(ir)
+	dec.UseNumber()
+	next := func() (v any, err error) { err = dec.Decode(&v); return }
+	return &jsonInputIter{next: next, ir: ir, fname: fname}
+}
+
+func (i *jsonInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	v, err := i.next()
+	if err != nil {
+		if err == io.EOF {
+			i.err = err
+			return nil, false
+		}
+		var offset *int64
+		var line *int
+		if err, ok := err.(*json.SyntaxError); ok {
+			err.Offset -= i.offset
+			offset, line = &err.Offset, &i.line
+		}
+		i.err = &jsonParseError{i.fname, i.ir.getContents(offset, line), i.line, err}
+		return i.err, true
+	}
+	if buf := i.ir.buf; buf != nil && buf.Len() >= 16*1024 {
+		i.offset += int64(buf.Len())
+		i.line += bytes.Count(buf.Bytes(), []byte{'\n'})
+		buf.Reset()
+	}
+	return v, true
+}
+
+func (i *jsonInputIter) Close() error {
+	i.err = io.EOF
+	return nil
+}
+
+func (i *jsonInputIter) Name() string {
+	return i.fname
+}
+
+func newStreamInputIter(r io.Reader, fname string) inputIter {
+	ir := newInputReader(r)
+	dec := json.NewDecoder(ir)
+	dec.UseNumber()
+	return &jsonInputIter{next: newJSONStream(dec).next, ir: ir, fname: fname}
+}
+
+type nullInputIter struct {
+	err error
+}
+
+func newNullInputIter() inputIter {
+	return &nullInputIter{}
+}
+
+func (i *nullInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	i.err = io.EOF
+	return nil, true
+}
+
+func (i *nullInputIter) Close() error {
+	i.err = io.EOF
+	return nil
+}
+
+func (*nullInputIter) Name() string {
+	return ""
+}
+
+type filesInputIter struct {
+	newIter func(io.Reader, string) inputIter
+	fnames  []string
+	stdin   io.Reader
+	iter    inputIter
+	file    io.Reader
+	err     error
+}
+
+func newFilesInputIter(
+	newIter func(io.Reader, string) inputIter, fnames []string, stdin io.Reader,
+) inputIter {
+	return &filesInputIter{newIter: newIter, fnames: fnames, stdin: stdin}
+}
+
+func (i *filesInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	for {
+		if i.file == nil {
+			if len(i.fnames) == 0 {
+				i.err = io.EOF
+				if i.iter != nil {
+					i.iter.Close()
+					i.iter = nil
+				}
+				return nil, false
+			}
+			fname := i.fnames[0]
+			i.fnames = i.fnames[1:]
+			if fname == "-" && i.stdin != nil {
+				i.file, fname = i.stdin, "<stdin>"
+			} else {
+				file, err := os.Open(fname)
+				if err != nil {
+					return err, true
+				}
+				i.file = file
+			}
+			if i.iter != nil {
+				i.iter.Close()
+			}
+			i.iter = i.newIter(i.file, fname)
+		}
+		if v, ok := i.iter.Next(); ok {
+			return v, ok
+		}
+		if r, ok := i.file.(io.Closer); ok && i.file != i.stdin {
+			r.Close()
+		}
+		i.file = nil
+	}
+}
+
+func (i *filesInputIter) Close() error {
+	if i.file != nil {
+		if r, ok := i.file.(io.Closer); ok && i.file != i.stdin {
+			r.Close()
+		}
+		i.file = nil
+		i.err = io.EOF
+	}
+	return nil
+}
+
+func (i *filesInputIter) Name() string {
+	if i.iter != nil {
+		return i.iter.Name()
+	}
+	return ""
+}
+
+type rawInputIter struct {
+	r     *bufio.Reader
+	fname string
+	err   error
+}
+
+func newRawInputIter(r io.Reader, fname string) inputIter {
+	return &rawInputIter{r: bufio.NewReader(r), fname: fname}
+}
+
+func (i *rawInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	line, err := i.r.ReadString('\n')
+	if err != nil {
+		i.err = err
+		if err != io.EOF {
+			return err, true
+		}
+		if line == "" {
+			return nil, false
+		}
+	}
+	return strings.TrimSuffix(line, "\n"), true
+}
+
+func (i *rawInputIter) Close() error {
+	i.err = io.EOF
+	return nil
+}
+
+func (i *rawInputIter) Name() string {
+	return i.fname
+}
+
+type slurpInputIter struct {
+	iter inputIter
+	err  error
+}
+
+func newSlurpInputIter(iter inputIter) inputIter {
+	return &slurpInputIter{iter: iter}
+}
+
+func (i *slurpInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	var vs []any
+	var v any
+	var ok bool
+	for {
+		v, ok = i.iter.Next()
+		if !ok {
+			i.err = io.EOF
+			return vs, true
+		}
+		if i.err, ok = v.(error); ok {
+			return i.err, true
+		}
+		vs = append(vs, v)
+	}
+}
+
+func (i *slurpInputIter) Close() error {
+	if i.iter != nil {
+		i.iter.Close()
+		i.iter = nil
+		i.err = io.EOF
+	}
+	return nil
+}
+
+func (i *slurpInputIter) Name() string {
+	return i.iter.Name()
+}
+
+type readAllIter struct {
+	r     io.Reader
+	fname string
+	err   error
+}
+
+func newReadAllIter(r io.Reader, fname string) inputIter {
+	return &readAllIter{r: r, fname: fname}
+}
+
+func (i *readAllIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	i.err = io.EOF
+	cnt, err := io.ReadAll(i.r)
+	if err != nil {
+		return err, true
+	}
+	return string(cnt), true
+}
+
+func (i *readAllIter) Close() error {
+	i.err = io.EOF
+	return nil
+}
+
+func (i *readAllIter) Name() string {
+	return i.fname
+}
+
+type slurpRawInputIter struct {
+	iter inputIter
+	err  error
+}
+
+func newSlurpRawInputIter(iter inputIter) inputIter {
+	return &slurpRawInputIter{iter: iter}
+}
+
+func (i *slurpRawInputIter) Next() (any, bool) {
+	if i.err != nil {
+		return nil, false
+	}
+	var vs []string
+	var v any
+	var ok bool
+	for {
+		v, ok = i.iter.Next()
+		if !ok {
+			i.err = io.EOF
+			return strings.Join(vs, ""), true
+		}
+		if i.err, ok = v.(error); ok {
+			return i.err, true
+		}
+		vs = append(vs, v.(string))
+	}
+}
+
+func (i *slurpRawInputIter) Close() error {
+	if i.iter != nil {
+		i.iter.Close()
+		i.iter = nil
+		i.err = io.EOF
+	}
+	return nil
+}
+
+func (i *slurpRawInputIter) Name() string {
+	return i.iter.Name()
+}

gojq-extended/marshaler.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"strings"
+)
+
+type marshaler interface {
+	marshal(any, io.Writer) error
+}
+
+type rawMarshaler struct {
+	m        marshaler
+	checkNul bool
+}
+
+func (m *rawMarshaler) marshal(v any, w io.Writer) error {
+	if s, ok := v.(string); ok {
+		if m.checkNul && strings.ContainsRune(s, '\x00') {
+			return fmt.Errorf("cannot output a string containing NUL character: %q", s)
+		}
+		_, err := w.Write([]byte(s))
+		return err
+	}
+	return m.m.marshal(v, w)
+}

gojq-extended/run.go

@@ -0,0 +1,13 @@
+package main
+
+import "os"
+
+// Run gojq.
+func main() {
+	cli := cli{
+		inStream:  os.Stdin,
+		outStream: os.Stdout,
+		errStream: os.Stderr,
+	}
+	os.Exit(cli.run(os.Args[1:]))
+}

gojq-extended/stream.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+)
+
+type jsonStream struct {
+	dec    *json.Decoder
+	path   []any
+	states []int
+}
+
+func newJSONStream(dec *json.Decoder) *jsonStream {
+	return &jsonStream{dec: dec, states: []int{jsonStateTopValue}, path: []any{}}
+}
+
+const (
+	jsonStateTopValue = iota
+	jsonStateArrayStart
+	jsonStateArrayValue
+	jsonStateArrayEnd
+	jsonStateArrayEmptyEnd
+	jsonStateObjectStart
+	jsonStateObjectKey
+	jsonStateObjectValue
+	jsonStateObjectEnd
+	jsonStateObjectEmptyEnd
+)
+
+func (s *jsonStream) next() (any, error) {
+	switch s.states[len(s.states)-1] {
+	case jsonStateArrayEnd, jsonStateObjectEnd:
+		s.path = s.path[:len(s.path)-1]
+		fallthrough
+	case jsonStateArrayEmptyEnd, jsonStateObjectEmptyEnd:
+		s.states = s.states[:len(s.states)-1]
+	}
+	if s.dec.More() {
+		switch s.states[len(s.states)-1] {
+		case jsonStateArrayValue:
+			s.path[len(s.path)-1] = s.path[len(s.path)-1].(int) + 1
+		case jsonStateObjectValue:
+			s.path = s.path[:len(s.path)-1]
+		}
+	}
+	for {
+		token, err := s.dec.Token()
+		if err != nil {
+			if err == io.EOF && s.states[len(s.states)-1] != jsonStateTopValue {
+				err = io.ErrUnexpectedEOF
+			}
+			return nil, err
+		}
+		if d, ok := token.(json.Delim); ok {
+			switch d {
+			case '[', '{':
+				switch s.states[len(s.states)-1] {
+				case jsonStateArrayStart:
+					s.states[len(s.states)-1] = jsonStateArrayValue
+				case jsonStateObjectKey:
+					s.states[len(s.states)-1] = jsonStateObjectValue
+				}
+				if d == '[' {
+					s.states = append(s.states, jsonStateArrayStart)
+					s.path = append(s.path, 0)
+				} else {
+					s.states = append(s.states, jsonStateObjectStart)
+				}
+			case ']':
+				if s.states[len(s.states)-1] == jsonStateArrayStart {
+					s.states[len(s.states)-1] = jsonStateArrayEmptyEnd
+					s.path = s.path[:len(s.path)-1]
+					return []any{s.copyPath(), []any{}}, nil
+				}
+				s.states[len(s.states)-1] = jsonStateArrayEnd
+				return []any{s.copyPath()}, nil
+			case '}':
+				if s.states[len(s.states)-1] == jsonStateObjectStart {
+					s.states[len(s.states)-1] = jsonStateObjectEmptyEnd
+					return []any{s.copyPath(), map[string]any{}}, nil
+				}
+				s.states[len(s.states)-1] = jsonStateObjectEnd
+				return []any{s.copyPath()}, nil
+			default:
+				panic(d)
+			}
+		} else {
+			switch s.states[len(s.states)-1] {
+			case jsonStateArrayStart:
+				s.states[len(s.states)-1] = jsonStateArrayValue
+				fallthrough
+			case jsonStateArrayValue:
+				return []any{s.copyPath(), token}, nil
+			case jsonStateObjectStart, jsonStateObjectValue:
+				s.states[len(s.states)-1] = jsonStateObjectKey
+				s.path = append(s.path, token)
+			case jsonStateObjectKey:
+				s.states[len(s.states)-1] = jsonStateObjectValue
+				return []any{s.copyPath(), token}, nil
+			default:
+				s.states[len(s.states)-1] = jsonStateTopValue
+				return []any{s.copyPath(), token}, nil
+			}
+		}
+	}
+}
+
+func (s *jsonStream) copyPath() []any {
+	path := make([]any, len(s.path))
+	copy(path, s.path)
+	return path
+}

run.sh

@@ -2,15 +2,33 @@
 
 set -eu
 
-SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
-cd "$SCRIPT_DIR/tool"
+WORKING_DIRECTORY=$(pwd)
+TOOL_DIR="$(cd -- "$(dirname -- "$0")" && pwd)/tool"
+cd "${WORKING_DIRECTORY}"
 
 JQ=${JQ:-jq}
-export JQ_FLAVOR=${JQ_FLAVOR:-${JQ}}
+export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"}
 
 run() {
-  ${JQ} -nr -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath ./dropins)/${JQ_FLAVOR}"  \
-    --slurpfile exportFile "${EXPORT_FILE:-export.json}" \
+  if [ -d "${TOOL_DIR}/lib/stubs/${JQ_FLAVOR}" ]; then 
+    STUBS_DIR="${TOOL_DIR}/lib/stubs/${JQ_FLAVOR}" 
+  else
+    STUBS_DIR="${TOOL_DIR}/lib/stubs/jq" 
+  fi
+
+  FILES_ARGS=()
+  if [ "${EXPORT_FILE:-}" != ""  ] && [ -f "${EXPORT_FILE}" ]; then
+    FILES_ARGS+=(--arg exportFileName "${EXPORT_FILE:-}")
+    if [ "${JQ_FLAVOR}" != "gojq-extended" ]; then
+      FILES_ARGS+=(--slurpfile exportFile "${EXPORT_FILE}")
+    fi
+  else
+    FILES_ARGS+=(--argjson exportFileName null)
+    FILES_ARGS+=(--argjson exportFile null)
+  fi
+
+  ${JQ} -nr -L "$(realpath "${TOOL_DIR}")" -L "$(realpath "${TOOL_DIR}/lib")" -L "$(realpath "${STUBS_DIR}")"  \
+    "${FILES_ARGS[@]}" \
     'include "main"; main' \
     --args -- "$@"
 }

runTests.sh

@@ -3,14 +3,21 @@
 SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
 cd "${SCRIPT_DIR}/tool" || return
 
+JQ=${JQ:-jq}
+export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"}
 export JQ_TYPECHECKING=1
 
+if [ ! -d "./lib/stubs/${JQ_FLAVOR}" ]; then 
+  STUBS_DIR="./lib/stubs/${JQ_FLAVOR}" 
+else
+  STUBS_DIR="./lib/stubs/jq" 
+fi
+
 runTests() {
-  JQ=${JQ:-jq}
-  export JQ_FLAVOR=${JQ_FLAVOR:-jq}
   echo "Running Tests with JQ=${JQ}"
+  export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"}
   ${JQ} -nr \
-    -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath ./dropins)/${JQ_FLAVOR}" \
+    -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath "${STUBS_DIR}")" \
     "include \"tests\"; testsMain"
 }
 

tool/journalUtils.jq

@@ -1,5 +1,3 @@
-include "dropins";
-
 import "lib/utilsLib" as utilsLib;
 import "lib/stringLib" as stringLib;
 import "lib/numberLib" as numberLib;
@@ -11,7 +9,7 @@ def formatExperienceTitle:
 
 def formatDose($dose; $unit; $isEstimate; $standardDeviation):
   if $dose == null then
-    "Unknown"
+    "Unknown \($unit)"
   else 
     (if $isEstimate then "~" else "" end) as $estimate |
     (if $standardDeviation != null then "±\($standardDeviation)" else "" end) as $standardDeviation |
@@ -34,7 +32,8 @@ def formatIngestionDose($customUnits):
     " (" +
     formatDose($customUnit.dose; $unit; $customUnit.isEstimate; $customUnit.estimatedDoseStandardDeviation) +
     " * " +
-    formatDose($ingestion.dose; $customUnit.unit; $ingestion.isDoseAnEstimate; $ingestion.estimatedDoseStandardDeviation)
+    formatDose($ingestion.dose; $customUnit.unit; $ingestion.isDoseAnEstimate; $ingestion.estimatedDoseStandardDeviation) +
+    ")"
   end;
 
 def formatIngestionTime:
@@ -68,15 +67,4 @@ def formatIngestionROA($customUnits): formatIngestionROA($customUnits; {});
 
 def formatIngestionInfo:
   . as $ingestionInfo |
-  if $ingestionInfo.dose == null then
-    "Unknown \($ingestionInfo.unit)"
-  else
-    if $ingestionInfo.isEstimate then "~" else "" end +
-    "\($ingestionInfo.dose * 100 | round / 100)" +
-    if $ingestionInfo.standardDeviation != null then
-      "±\($ingestionInfo.standardDeviation)"
-    else "" end +
-
-    if $ingestionInfo.isUnknown then "+ Unknown" else "" end +
-    " \($ingestionInfo.unit)"
-  end;
+  formatDose(.dose; .unit; .isEstimate; .standardDeviation);

tool/lib/.jq-lsp.jq

@@ -0,0 +1,2 @@
+def _readFile($filename): empty;
+def _writeFileString($filename): empty;

tool/lib/gojqExtendedLib.jq

@@ -0,0 +1,7 @@
+include "stubs";
+
+def checkSupported:
+  $ENV.JQ_FLAVOR == "gojq-extended";
+
+def readFile($filename): _readFile($filename);
+def writeFileString($filename): _writeFileString($filename);

tool/lib/journalLib.jq

@@ -96,9 +96,7 @@ def ingestionsByConsumer:
     $consumerNames[] as $consumerName |
       {
         key: $consumerName,
-        value: $ingestions | map(select(
-          . | ingestionConsumerName == $consumerName
-        )),  
+        value: $ingestions | map(select((. | ingestionConsumerName) == $consumerName)),  
       }
   ] | from_entries;
 
@@ -167,7 +165,7 @@ def experienceStats($customUnits):
     $ingestion |
     .substanceName as $name |
     .administrationRoute as $administrationRoute |
-    . | ingestionConsumerName as $consumerName |
+    (. | ingestionConsumerName) as $consumerName |
 
     $stats |
       .[$consumerName].[$ingestion.substanceName].[$administrationRoute] as $ingestionStats |

tool/lib/journalLibTests.jq

@@ -1,5 +1,3 @@
-include "dropins";
-
 import "testLib" as testLib;
 import "journalLib" as journalLib;
 

tool/lib/journalTypes.jq

@@ -1,5 +1,3 @@
-include "dropins";
-
 import "typeLib" as typeLib;
 
 def ensureAdministrationRoute:

tool/lib/stubs/jq/stubs.jq

@@ -0,0 +1,2 @@
+def _readFile($filename): error("_readFile only supported in gojq-extended");
+def _writeFileString($filename): error("_writeFileString only supported in gojq-extended");

tool/lib/utilsLib.jq

@@ -1,5 +1,3 @@
-include "dropins";
-
 import "stringLib" as stringLib;
 
 def debugLog($target; $value):

tool/main.jq

@@ -1,11 +1,10 @@
-include "dropins";
-
 import "lib/typeLib" as typeLib;
 import "lib/argsLib" as argsLib;
 import "lib/stringLib" as stringLib;
 import "lib/tableLib" as tableLib;
 import "lib/utilsLib" as utilsLib;
 import "lib/journalLib" as journalLib;
+import "lib/gojqExtendedLib" as gojqExtendedLib;
 import "lib/journalTypes" as journalTypes;
 
 import "journalUtils" as journalUtils;
@@ -22,13 +21,14 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle):
   ($ingestions | journalLib::ingestionsSubstanceNames) as $substanceNames |
   ($ingestions | journalLib::ingestionsByConsumer) as $ingestionsByConsumer |
 
+  ($ingestionsByConsumer | keys) as $consumerNames |
+
   "" as $experienceStatsText |
   $experienceStatsText | 
     if $withTitle then 
       . += ($experience | journalUtils::formatExperienceTitle | . + "\n")
     end | . as $experienceStatsText |
 
-  ($ingestionsByConsumer | keys) as $consumerNames |
   reduce $consumerNames[] as $consumerName ($experienceStatsText;
     . as $experienceStatsText |
     $experienceStatsText |
@@ -38,7 +38,7 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle):
 
     ($ingestionsByConsumer[$consumerName] | journalLib::ingestionsSubstanceNames) as $consumerSubstanceNames |
 
-    $experienceStatsText | reduce $substanceNames[] as $substanceName (.; 
+    $experienceStatsText | reduce $consumerSubstanceNames[] as $substanceName (.; 
       . as $experienceStatsText |
 
       ($stats.[$consumerName].[$substanceName] | keys) as $ingestionMethods |
@@ -52,8 +52,22 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle):
         .key as $ingestionMethod |
         .value as $ingestionInfo |
 
+        def formatIngestionInfo:
+          if $ingestionInfo.dose == null then
+            "Unknown \($ingestionInfo.unit)"
+          else
+            if $ingestionInfo.isEstimate then "~" else "" end +
+            "\($ingestionInfo.dose * 100 | round / 100)" +
+            if $ingestionInfo.standardDeviation != null then
+              "±\($ingestionInfo.standardDeviation)"
+            else "" end +
+
+            if $ingestionInfo.isUnknown then "+ Unknown" else "" end +
+            " \($ingestionInfo.unit)"
+          end;
+
         $experienceStatsText | 
-          . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($ingestionInfo | journalUtils::formatIngestionInfo)\n" |
+          . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($ingestionInfo | formatIngestionInfo)\n" |
           . as $experienceStatsText | 
           
         $experienceStatsText
@@ -133,14 +147,47 @@ def main:
       ""
     ] | join("\n") | halt_error(1);
 
-  $ARGS.named["exportFile"][0] as $exportData |
-  $exportData | journalTypes::ensureExportData |
-
   ($ARGS | argsLib::parseArgs) as $parsedArgs |
 
   $parsedArgs.nonArgs[0] as $program |
   ($parsedArgs | .nonArgs |= $parsedArgs.nonArgs[1:]) as $parsedArgs |
 
+  if gojqExtendedLib::checkSupported then
+    if $parsedArgs.longArgs | has("export-file") then
+      $parsedArgs.longArgs["export-file"] 
+    else 
+      $ARGS.named["exportFileName"] 
+    end | . as $exportFileName |
+    {
+      name: $exportFileName,
+      content: gojqExtendedLib::readFile($exportFileName) | fromjson
+    }
+  else
+    if $parsedArgs.longArgs | has("export-file") then
+      debug("--export-file was provided but this version of jq doesn't support reading files" + 
+        "using EXPORT_FILE instead")
+    end | 
+    {
+      name: $ARGS.named["exportFileName"], 
+      content: $ARGS.named["exportFile"][0]
+    }
+  end | . as $exportFile |
+
+  $exportFile.name as $exportFileName |
+  $exportFile.content as $exportData |
+
+  if $exportData == null then
+    if gojqExtendedLib::checkSupported then
+      "please set EXPORT_FILE= or --export-file to a valid psychonaut journal export file\n" |
+        halt_error(1)
+    else 
+      "please set EXPORT_FILE= to a valid psychonaut journal export file\n" |
+        halt_error(1)
+    end
+  end |
+
+  $exportData | journalTypes::ensureExportData |
+
   if $program == null then
     if any($parsedArgs.shortArgs[]; . == "h") then usage end |
     if ($parsedArgs.longArgs | has("help")) then usage end |

tool/tests.jq

@@ -1,5 +1,3 @@
-include "dropins";
-
 import "lib/testLib" as testLib;
 import "lib/typeLib" as typeLib;
 import "lib/journalLibTests" as journalLibTests;