diff --git a/.gitignore b/.gitignore index 54a02e8..8440023 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ export.json .dev.env -dev \ No newline at end of file +dev +*.env \ No newline at end of file diff --git a/gojq-extended/.gitignore b/gojq-extended/.gitignore new file mode 100644 index 0000000..86adbca --- /dev/null +++ b/gojq-extended/.gitignore @@ -0,0 +1 @@ +gojq-extended \ No newline at end of file diff --git a/gojq-extended/LICENSE b/gojq-extended/LICENSE new file mode 100644 index 0000000..8dacc6c --- /dev/null +++ b/gojq-extended/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2024 chaos +Copyright (c) 2019-2024 itchyny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gojq-extended/LICENSE.orig b/gojq-extended/LICENSE.orig new file mode 100644 index 0000000..fe59004 --- /dev/null +++ b/gojq-extended/LICENSE.orig @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019-2024 itchyny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gojq-extended/cli.go b/gojq-extended/cli.go new file mode 100644 index 0000000..3f337ce --- /dev/null +++ b/gojq-extended/cli.go @@ -0,0 +1,474 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "runtime" + "strings" + + "github.com/mattn/go-isatty" + + "github.com/itchyny/gojq" +) + +const name = "gojq" + +const version = "0.12.16" + +var revision = "HEAD" + +const ( + exitCodeOK = iota + exitCodeFalsyErr + exitCodeFlagParseErr + exitCodeCompileErr + exitCodeNoValueErr + exitCodeDefaultErr +) + +type cli struct { + inStream io.Reader + outStream io.Writer + errStream io.Writer + + outputRaw bool + outputRaw0 bool + outputJoin bool + outputCompact bool + outputIndent *int + outputTab bool + inputRaw bool + inputStream bool + inputSlurp bool + + argnames []string + argvalues []any + + exitCodeError error +} + +type flagopts struct { + OutputRaw bool `short:"r" long:"raw-output" description:"output raw strings"` + OutputRaw0 bool `long:"raw-output0" description:"implies -r with NUL character delimiter"` + OutputJoin bool `short:"j" long:"join-output" description:"implies -r with no newline delimiter"` + OutputCompact bool `short:"c" long:"compact-output" description:"output without pretty-printing"` + OutputIndent *int `long:"indent" description:"number of spaces for indentation"` + OutputTab bool `long:"tab" description:"use tabs for indentation"` + OutputColor bool `short:"C" long:"color-output" description:"output with colors even if piped"` + OutputMono bool `short:"M" long:"monochrome-output" description:"output without colors"` + InputNull bool `short:"n" long:"null-input" description:"use null as input value"` + InputRaw bool `short:"R" long:"raw-input" description:"read input as raw strings"` + InputStream bool `long:"stream" description:"parse input in stream fashion"` + InputSlurp bool `short:"s" long:"slurp" description:"read all inputs into an array"` + FromFile bool `short:"f" long:"from-file" description:"load query from file"` + ModulePaths []string `short:"L" description:"directory to search modules from"` + Arg map[string]string `long:"arg" description:"set a string value to a variable"` + ArgJSON map[string]string `long:"argjson" description:"set a JSON value to a variable"` + SlurpFile map[string]string `long:"slurpfile" description:"set the JSON contents of a file to a variable"` + RawFile map[string]string `long:"rawfile" description:"set the contents of a file to a variable"` + Args []any `long:"args" positional:"" description:"consume remaining arguments as positional string values"` + JSONArgs []any `long:"jsonargs" positional:"" description:"consume remaining arguments as positional JSON values"` + ExitStatus bool `short:"e" long:"exit-status" description:"exit 1 when the last value is false or null"` + Version bool `short:"v" long:"version" description:"display version information"` + Help bool `short:"h" long:"help" description:"display this help information"` +} + +var addDefaultModulePaths = true + +func (cli *cli) run(args []string) int { + if err := cli.runInternal(args); err != nil { + if _, ok := err.(interface{ isEmptyError() }); !ok { + fmt.Fprintf(cli.errStream, "%s: %s\n", name, err) + } + if err, ok := err.(interface{ ExitCode() int }); ok { + return err.ExitCode() + } + return exitCodeDefaultErr + } + return exitCodeOK +} + +func (cli *cli) runInternal(args []string) (err error) { + var opts flagopts + args, err = parseFlags(args, &opts) + if err != nil { + return &flagParseError{err} + } + if opts.Help { + fmt.Fprintf(cli.outStream, `%[1]s - Go implementation of jq + +Version: %s (rev: %s/%s) + +Synopsis: + %% echo '{"foo": 128}' | %[1]s '.foo' + +Usage: + %[1]s [OPTIONS] + +`, + name, version, revision, runtime.Version()) + fmt.Fprintln(cli.outStream, formatFlags(&opts)) + return nil + } + if opts.Version { + fmt.Fprintf(cli.outStream, "%s %s (rev: %s/%s)\n", name, version, revision, runtime.Version()) + return nil + } + cli.outputRaw, cli.outputRaw0, cli.outputJoin, + cli.outputCompact, cli.outputIndent, cli.outputTab = + opts.OutputRaw, opts.OutputRaw0, opts.OutputJoin, + opts.OutputCompact, opts.OutputIndent, opts.OutputTab + defer func(x bool) { noColor = x }(noColor) + if opts.OutputColor || opts.OutputMono { + noColor = opts.OutputMono + } else if os.Getenv("NO_COLOR") != "" || os.Getenv("TERM") == "dumb" { + noColor = true + } else { + f, ok := cli.outStream.(interface{ Fd() uintptr }) + noColor = !(ok && (isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd()))) + } + if !noColor { + if colors := os.Getenv("GOJQ_COLORS"); colors != "" { + if err := setColors(colors); err != nil { + return err + } + } + } + if i := cli.outputIndent; i != nil { + if *i > 9 { + return fmt.Errorf("too many indentation count: %d", *i) + } else if *i < 0 { + return fmt.Errorf("negative indentation count: %d", *i) + } + } + + cli.inputRaw, cli.inputStream, cli.inputSlurp = + opts.InputRaw, opts.InputStream, opts.InputSlurp + for k, v := range opts.Arg { + cli.argnames = append(cli.argnames, "$"+k) + cli.argvalues = append(cli.argvalues, v) + } + for k, v := range opts.ArgJSON { + val, _ := newJSONInputIter(strings.NewReader(v), "$"+k).Next() + if err, ok := val.(error); ok { + return err + } + cli.argnames = append(cli.argnames, "$"+k) + cli.argvalues = append(cli.argvalues, val) + } + for k, v := range opts.SlurpFile { + val, err := slurpFile(v) + if err != nil { + return err + } + cli.argnames = append(cli.argnames, "$"+k) + cli.argvalues = append(cli.argvalues, val) + } + for k, v := range opts.RawFile { + val, err := os.ReadFile(v) + if err != nil { + return err + } + cli.argnames = append(cli.argnames, "$"+k) + cli.argvalues = append(cli.argvalues, string(val)) + } + named := make(map[string]any, len(cli.argnames)) + for i, name := range cli.argnames { + named[name[1:]] = cli.argvalues[i] + } + positional := opts.Args + for i, v := range opts.JSONArgs { + if v != nil { + val, _ := newJSONInputIter(strings.NewReader(v.(string)), "--jsonargs").Next() + if err, ok := val.(error); ok { + return err + } + if i < len(positional) { + positional[i] = val + } else { + positional = append(positional, val) + } + } + } + cli.argnames = append(cli.argnames, "$ARGS") + cli.argvalues = append(cli.argvalues, map[string]any{ + "named": named, + "positional": positional, + }) + var arg, fname string + if opts.FromFile { + if len(args) == 0 { + return errors.New("expected a query file for flag `-f'") + } + src, err := os.ReadFile(args[0]) + if err != nil { + return err + } + arg, args, fname = string(src), args[1:], args[0] + } else if len(args) == 0 { + arg = "." + } else { + arg, args, fname = strings.TrimSpace(args[0]), args[1:], "" + } + if opts.ExitStatus { + cli.exitCodeError = &exitCodeError{exitCodeNoValueErr} + defer func() { + if _, ok := err.(interface{ ExitCode() int }); !ok { + err = cli.exitCodeError + } + }() + } + query, err := gojq.Parse(arg) + if err != nil { + return &queryParseError{fname, arg, err} + } + modulePaths := opts.ModulePaths + if len(modulePaths) == 0 && addDefaultModulePaths { + modulePaths = []string{"~/.jq", "$ORIGIN/../lib/gojq", "$ORIGIN/../lib"} + } + iter := cli.createInputIter(args) + defer iter.Close() + code, err := gojq.Compile(query, + gojq.WithModuleLoader(gojq.NewModuleLoader(modulePaths)), + gojq.WithEnvironLoader(os.Environ), + gojq.WithVariables(cli.argnames), + gojq.WithFunction("debug", 0, 0, + func(errStream io.Writer) func(any, []any) any { + indent := 2 + if cli.outputCompact { + indent = 0 + } else if cli.outputTab { + indent = 1 + } else if i := cli.outputIndent; i != nil { + indent = *i + } + + return func(v any, _ []any) any { + if err := newEncoder(false, indent). + marshal([]any{"DEBUG:", v}, cli.errStream); err != nil { + return err + } + if _, err := cli.errStream.Write([]byte{'\n'}); err != nil { + return err + } + return v + } + }(cli.errStream), + ), + gojq.WithFunction("stderr", 0, 0, + func(errStream io.Writer) func(any, []any) any { + return func(v any, _ []any) any { + if err := (&rawMarshaler{m: newEncoder(false, 0)}). + marshal(v, cli.errStream); err != nil { + return err + } + return v + } + }(cli.errStream), + ), + gojq.WithFunction("_readFile", 1, 1, + func(_input any, args []any) any { + filename := args[0].(string) + + info, err := os.Stat(filename) + if err != nil { + return fmt.Errorf("file not found") + } + + if info.IsDir() { + return fmt.Errorf("file is a directory") + } + + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("file could not be read") + } + return string(content) + }, + ), + gojq.WithFunction("_writeFileString", 1, 1, + func(input any, args []any) any { + filename := args[0].(string) + content := "" + + switch input := input.(type) { + case string: + content = input + default: + return fmt.Errorf("invalid type passed to _writeFileString") + } + + file, err := os.Create(filename) + if err != nil { + return err + } + + _, err = file.Write([]byte(content)) + if err != nil { + return err + } + err = file.Close() + if err != nil { + return err + } + + return nil + }, + ), + gojq.WithFunction("input_filename", 0, 0, + func(iter inputIter) func(any, []any) any { + return func(any, []any) any { + if fname := iter.Name(); fname != "" && (len(args) > 0 || !opts.InputNull) { + return fname + } + return nil + } + }(iter), + ), + gojq.WithInputIter(iter), + ) + if err != nil { + if err, ok := err.(interface { + QueryParseError() (string, string, error) + }); ok { + name, query, err := err.QueryParseError() + return &queryParseError{name, query, err} + } + if err, ok := err.(interface { + JSONParseError() (string, string, error) + }); ok { + fname, contents, err := err.JSONParseError() + return &compileError{&jsonParseError{fname, contents, 0, err}} + } + return &compileError{err} + } + if opts.InputNull { + iter = newNullInputIter() + } + return cli.process(iter, code) +} + +func slurpFile(name string) (any, error) { + iter := newSlurpInputIter( + newFilesInputIter(newJSONInputIter, []string{name}, nil), + ) + defer iter.Close() + val, _ := iter.Next() + if err, ok := val.(error); ok { + return nil, err + } + return val, nil +} + +func (cli *cli) createInputIter(args []string) (iter inputIter) { + var newIter func(io.Reader, string) inputIter + switch { + case cli.inputRaw: + if cli.inputSlurp { + newIter = newReadAllIter + } else { + newIter = newRawInputIter + } + case cli.inputStream: + newIter = newStreamInputIter + default: + newIter = newJSONInputIter + } + if cli.inputSlurp { + defer func() { + if cli.inputRaw { + iter = newSlurpRawInputIter(iter) + } else { + iter = newSlurpInputIter(iter) + } + }() + } + if len(args) == 0 { + return newIter(cli.inStream, "") + } + return newFilesInputIter(newIter, args, cli.inStream) +} + +func (cli *cli) process(iter inputIter, code *gojq.Code) error { + var err error + for { + v, ok := iter.Next() + if !ok { + break + } + if e, ok := v.(error); ok { + fmt.Fprintf(cli.errStream, "%s: %s\n", name, e) + err = e + continue + } + if e := cli.printValues(code.Run(v, cli.argvalues...)); e != nil { + if e, ok := e.(*gojq.HaltError); ok { + if v := e.Value(); v != nil { + if str, ok := v.(string); ok { + cli.errStream.Write([]byte(str)) + } else { + bs, _ := gojq.Marshal(v) + cli.errStream.Write(bs) + cli.errStream.Write([]byte{'\n'}) + } + } + err = e + break + } + fmt.Fprintf(cli.errStream, "%s: %s\n", name, e) + err = e + } + } + if err != nil { + return &emptyError{err} + } + return nil +} + +func (cli *cli) printValues(iter gojq.Iter) error { + m := cli.createMarshaler() + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return err + } + + if err := m.marshal(v, cli.outStream); err != nil { + return err + } + if cli.exitCodeError != nil { + if v == nil || v == false { + cli.exitCodeError = &exitCodeError{exitCodeFalsyErr} + } else { + cli.exitCodeError = &exitCodeError{exitCodeOK} + } + } + if cli.outputRaw0 { + cli.outStream.Write([]byte{'\x00'}) + } else if !cli.outputJoin { + cli.outStream.Write([]byte{'\n'}) + } + } + return nil +} + +func (cli *cli) createMarshaler() marshaler { + indent := 2 + if cli.outputCompact { + indent = 0 + } else if cli.outputTab { + indent = 1 + } else if i := cli.outputIndent; i != nil { + indent = *i + } + f := newEncoder(cli.outputTab, indent) + if cli.outputRaw || cli.outputRaw0 || cli.outputJoin { + return &rawMarshaler{f, cli.outputRaw0} + } + return f +} diff --git a/gojq-extended/color.go b/gojq-extended/color.go new file mode 100644 index 0000000..4f08a40 --- /dev/null +++ b/gojq-extended/color.go @@ -0,0 +1,64 @@ +package main + +import ( + "bytes" + "fmt" + "strings" +) + +var noColor bool + +func newColor(c string) []byte { + return []byte("\x1b[" + c + "m") +} + +func setColor(buf *bytes.Buffer, color []byte) { + if !noColor { + buf.Write(color) + } +} + +var ( + resetColor = newColor("0") // Reset + nullColor = newColor("90") // Bright black + falseColor = newColor("33") // Yellow + trueColor = newColor("33") // Yellow + numberColor = newColor("36") // Cyan + stringColor = newColor("32") // Green + objectKeyColor = newColor("34;1") // Bold Blue + arrayColor = []byte(nil) // No color + objectColor = []byte(nil) // No color +) + +func validColor(x string) bool { + var num bool + for _, c := range x { + if '0' <= c && c <= '9' { + num = true + } else if c == ';' && num { + num = false + } else { + return false + } + } + return num +} + +func setColors(colors string) error { + var color string + for _, target := range []*[]byte{ + &nullColor, &falseColor, &trueColor, &numberColor, + &stringColor, &objectKeyColor, &arrayColor, &objectColor, + } { + color, colors, _ = strings.Cut(colors, ":") + if color != "" { + if !validColor(color) { + return fmt.Errorf("invalid color: %q", color) + } + *target = newColor(color) + } else { + *target = nil + } + } + return nil +} diff --git a/gojq-extended/encoder.go b/gojq-extended/encoder.go new file mode 100644 index 0000000..a96ae21 --- /dev/null +++ b/gojq-extended/encoder.go @@ -0,0 +1,267 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "math" + "math/big" + "sort" + "strconv" + "unicode/utf8" +) + +type encoder struct { + out io.Writer + w *bytes.Buffer + tab bool + indent int + depth int + buf [64]byte +} + +func newEncoder(tab bool, indent int) *encoder { + // reuse the buffer in multiple calls of marshal + return &encoder{w: new(bytes.Buffer), tab: tab, indent: indent} +} + +func (e *encoder) flush() error { + _, err := e.out.Write(e.w.Bytes()) + e.w.Reset() + return err +} + +func (e *encoder) marshal(v any, w io.Writer) error { + e.out = w + err := e.encode(v) + if ferr := e.flush(); ferr != nil && err == nil { + err = ferr + } + return err +} + +func (e *encoder) encode(v any) error { + switch v := v.(type) { + case nil: + e.write([]byte("null"), nullColor) + case bool: + if v { + e.write([]byte("true"), trueColor) + } else { + e.write([]byte("false"), falseColor) + } + case int: + e.write(strconv.AppendInt(e.buf[:0], int64(v), 10), numberColor) + case float64: + e.encodeFloat64(v) + case *big.Int: + e.write(v.Append(e.buf[:0], 10), numberColor) + case string: + e.encodeString(v, stringColor) + case []any: + if err := e.encodeArray(v); err != nil { + return err + } + case map[string]any: + if err := e.encodeObject(v); err != nil { + return err + } + default: + panic(fmt.Sprintf("invalid type: %[1]T (%[1]v)", v)) + } + if e.w.Len() > 8*1024 { + return e.flush() + } + return nil +} + +// ref: floatEncoder in encoding/json +func (e *encoder) encodeFloat64(f float64) { + if math.IsNaN(f) { + e.write([]byte("null"), nullColor) + return + } + f = min(max(f, -math.MaxFloat64), math.MaxFloat64) + format := byte('f') + if x := math.Abs(f); x != 0 && x < 1e-6 || x >= 1e21 { + format = 'e' + } + buf := strconv.AppendFloat(e.buf[:0], f, format, -1, 64) + if format == 'e' { + // clean up e-09 to e-9 + if n := len(buf); n >= 4 && buf[n-4] == 'e' && buf[n-3] == '-' && buf[n-2] == '0' { + buf[n-2] = buf[n-1] + buf = buf[:n-1] + } + } + e.write(buf, numberColor) +} + +// ref: encodeState#string in encoding/json +func (e *encoder) encodeString(s string, color []byte) { + if color != nil { + setColor(e.w, color) + } + e.w.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if ' ' <= b && b <= '~' && b != '"' && b != '\\' { + i++ + continue + } + if start < i { + e.w.WriteString(s[start:i]) + } + switch b { + case '"': + e.w.WriteString(`\"`) + case '\\': + e.w.WriteString(`\\`) + case '\b': + e.w.WriteString(`\b`) + case '\f': + e.w.WriteString(`\f`) + case '\n': + e.w.WriteString(`\n`) + case '\r': + e.w.WriteString(`\r`) + case '\t': + e.w.WriteString(`\t`) + default: + const hex = "0123456789abcdef" + e.w.WriteString(`\u00`) + e.w.WriteByte(hex[b>>4]) + e.w.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + e.w.WriteString(s[start:i]) + } + e.w.WriteString(`\ufffd`) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + e.w.WriteString(s[start:]) + } + e.w.WriteByte('"') + if color != nil { + setColor(e.w, resetColor) + } +} + +func (e *encoder) encodeArray(vs []any) error { + e.writeByte('[', arrayColor) + e.depth += e.indent + for i, v := range vs { + if i > 0 { + e.writeByte(',', arrayColor) + } + if e.indent != 0 { + e.writeIndent() + } + if err := e.encode(v); err != nil { + return err + } + } + e.depth -= e.indent + if len(vs) > 0 && e.indent != 0 { + e.writeIndent() + } + e.writeByte(']', arrayColor) + return nil +} + +func (e *encoder) encodeObject(vs map[string]any) error { + e.writeByte('{', objectColor) + e.depth += e.indent + type keyVal struct { + key string + val any + } + kvs := make([]keyVal, len(vs)) + var i int + for k, v := range vs { + kvs[i] = keyVal{k, v} + i++ + } + sort.Slice(kvs, func(i, j int) bool { + return kvs[i].key < kvs[j].key + }) + for i, kv := range kvs { + if i > 0 { + e.writeByte(',', objectColor) + } + if e.indent != 0 { + e.writeIndent() + } + e.encodeString(kv.key, objectKeyColor) + e.writeByte(':', objectColor) + if e.indent != 0 { + e.w.WriteByte(' ') + } + if err := e.encode(kv.val); err != nil { + return err + } + } + e.depth -= e.indent + if len(vs) > 0 && e.indent != 0 { + e.writeIndent() + } + e.writeByte('}', objectColor) + return nil +} + +func (e *encoder) writeIndent() { + e.w.WriteByte('\n') + if n := e.depth; n > 0 { + if e.tab { + e.writeIndentInternal(n, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t") + } else { + e.writeIndentInternal(n, " ") + } + } +} + +func (e *encoder) writeIndentInternal(n int, spaces string) { + if l := len(spaces); n <= l { + e.w.WriteString(spaces[:n]) + } else { + e.w.WriteString(spaces) + for n -= l; n > 0; n, l = n-l, l*2 { + if n < l { + l = n + } + e.w.Write(e.w.Bytes()[e.w.Len()-l:]) + } + } +} + +func (e *encoder) writeByte(b byte, color []byte) { + if color == nil { + e.w.WriteByte(b) + } else { + setColor(e.w, color) + e.w.WriteByte(b) + setColor(e.w, resetColor) + } +} + +func (e *encoder) write(bs []byte, color []byte) { + if color == nil { + e.w.Write(bs) + } else { + setColor(e.w, color) + e.w.Write(bs) + setColor(e.w, resetColor) + } +} diff --git a/gojq-extended/error.go b/gojq-extended/error.go new file mode 100644 index 0000000..0219f78 --- /dev/null +++ b/gojq-extended/error.go @@ -0,0 +1,225 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" + "unicode/utf8" + + "github.com/mattn/go-runewidth" + + "github.com/itchyny/gojq" +) + +type emptyError struct { + err error +} + +func (*emptyError) Error() string { + return "" +} + +func (*emptyError) isEmptyError() {} + +func (err *emptyError) ExitCode() int { + if err, ok := err.err.(interface{ ExitCode() int }); ok { + return err.ExitCode() + } + return exitCodeDefaultErr +} + +type exitCodeError struct { + code int +} + +func (err *exitCodeError) Error() string { + return "exit code: " + strconv.Itoa(err.code) +} + +func (*exitCodeError) isEmptyError() {} + +func (err *exitCodeError) ExitCode() int { + return err.code +} + +type flagParseError struct { + err error +} + +func (err *flagParseError) Error() string { + return err.err.Error() +} + +func (*flagParseError) ExitCode() int { + return exitCodeFlagParseErr +} + +type compileError struct { + err error +} + +func (err *compileError) Error() string { + return "compile error: " + err.err.Error() +} + +func (*compileError) ExitCode() int { + return exitCodeCompileErr +} + +type queryParseError struct { + fname, contents string + err error +} + +func (err *queryParseError) Error() string { + var offset int + var e *gojq.ParseError + if errors.As(err.err, &e) { + offset = e.Offset - len(e.Token) + 1 + } + linestr, line, column := getLineByOffset(err.contents, offset) + if err.fname != "" || containsNewline(err.contents) { + return fmt.Sprintf("invalid query: %s:%d\n%s %s", + err.fname, line, formatLineInfo(linestr, line, column), err.err) + } + return fmt.Sprintf("invalid query: %s\n %s\n %*c %s", + err.contents, linestr, column+1, '^', err.err) +} + +func (*queryParseError) ExitCode() int { + return exitCodeCompileErr +} + +type jsonParseError struct { + fname, contents string + line int + err error +} + +func (err *jsonParseError) Error() string { + var offset int + if err.err == io.ErrUnexpectedEOF { + offset = len(err.contents) + 1 + } else if e, ok := err.err.(*json.SyntaxError); ok { + offset = int(e.Offset) + } + linestr, line, column := getLineByOffset(err.contents, offset) + if line += err.line; line > 1 { + return fmt.Sprintf("invalid json: %s:%d\n%s %s", + err.fname, line, formatLineInfo(linestr, line, column), err.err) + } + return fmt.Sprintf("invalid json: %s\n %s\n %*c %s", + err.fname, linestr, column+1, '^', err.err) +} + +func getLineByOffset(str string, offset int) (linestr string, line, column int) { + ss := &stringScanner{str, 0} + for { + str, start, ok := ss.next() + if !ok { + offset -= start + break + } + line++ + linestr = str + if ss.offset >= offset { + offset -= start + break + } + } + offset = min(max(offset-1, 0), len(linestr)) + if offset > 48 { + skip := len(trimLastInvalidRune(linestr[:offset-48])) + linestr = linestr[skip:] + offset -= skip + } + linestr = trimLastInvalidRune(linestr[:min(64, len(linestr))]) + if offset < len(linestr) { + offset = len(trimLastInvalidRune(linestr[:offset])) + } else { + offset = len(linestr) + } + column = runewidth.StringWidth(linestr[:offset]) + return +} + +func getLineByLine(str string, line int) (linestr string) { + ss := &stringScanner{str, 0} + for { + str, _, ok := ss.next() + if !ok { + break + } + if line--; line == 0 { + linestr = str + break + } + } + if len(linestr) > 64 { + linestr = trimLastInvalidRune(linestr[:64]) + } + return +} + +func trimLastInvalidRune(s string) string { + for i := len(s) - 1; i >= 0 && i > len(s)-utf8.UTFMax; i-- { + if b := s[i]; b < utf8.RuneSelf { + return s[:i+1] + } else if utf8.RuneStart(b) { + if r, _ := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError { + return s[:i] + } + break + } + } + return s +} + +func formatLineInfo(linestr string, line, column int) string { + l := strconv.Itoa(line) + return fmt.Sprintf(" %s | %s\n %*c", l, linestr, column+len(l)+4, '^') +} + +type stringScanner struct { + str string + offset int +} + +func (ss *stringScanner) next() (line string, start int, ok bool) { + if ss.offset == len(ss.str) { + return + } + start, ok = ss.offset, true + line = ss.str[start:] + i := indexNewline(line) + if i < 0 { + ss.offset = len(ss.str) + return + } + line = line[:i] + if strings.HasPrefix(ss.str[start+i:], "\r\n") { + i++ + } + ss.offset += i + 1 + return +} + +// Faster than strings.ContainsAny(str, "\r\n"). +func containsNewline(str string) bool { + return strings.IndexByte(str, '\n') >= 0 || + strings.IndexByte(str, '\r') >= 0 +} + +// Faster than strings.IndexAny(str, "\r\n"). +func indexNewline(str string) (i int) { + if i = strings.IndexByte(str, '\n'); i >= 0 { + str = str[:i] + } + if j := strings.IndexByte(str, '\r'); j >= 0 { + i = j + } + return +} diff --git a/gojq-extended/flags.go b/gojq-extended/flags.go new file mode 100644 index 0000000..480d004 --- /dev/null +++ b/gojq-extended/flags.go @@ -0,0 +1,211 @@ +package main + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +func parseFlags(args []string, opts any) ([]string, error) { + rest := make([]string, 0, len(args)) + val := reflect.ValueOf(opts).Elem() + typ := val.Type() + longToValue := map[string]reflect.Value{} + longToPositional := map[string]struct{}{} + shortToValue := map[string]reflect.Value{} + for i, l := 0, val.NumField(); i < l; i++ { + if flag, ok := typ.Field(i).Tag.Lookup("long"); ok { + longToValue[flag] = val.Field(i) + if _, ok := typ.Field(i).Tag.Lookup("positional"); ok { + longToPositional[flag] = struct{}{} + } + } + if flag, ok := typ.Field(i).Tag.Lookup("short"); ok { + shortToValue[flag] = val.Field(i) + } + } + mapKeys := map[string]struct{}{} + var positionalVal reflect.Value + for i := 0; i < len(args); i++ { + arg := args[i] + var ( + val reflect.Value + ok bool + shortopts string + ) + if arg == "--" { + if positionalVal.IsValid() { + for _, arg := range args[i+1:] { + positionalVal.Set(reflect.Append(positionalVal, reflect.ValueOf(arg))) + } + } else { + rest = append(rest, args[i+1:]...) + } + break + } + if strings.HasPrefix(arg, "--") { + if val, ok = longToValue[arg[2:]]; !ok { + if j := strings.IndexByte(arg, '='); j >= 0 { + if val, ok = longToValue[arg[2:j]]; ok { + if val.Kind() == reflect.Bool { + return nil, fmt.Errorf("boolean flag `%s' cannot have an argument", arg[:j]) + } + args[i] = arg[j+1:] + arg = arg[:j] + i-- + } + } + if !ok { + return nil, fmt.Errorf("unknown flag `%s'", arg) + } + } + } else if len(arg) > 1 && arg[0] == '-' { + var skip bool + for i := 1; i < len(arg); i++ { + opt := arg[i : i+1] + if val, ok = shortToValue[opt]; ok { + if val.Kind() != reflect.Bool { + break + } + } else if !("A" <= opt && opt <= "Z" || "a" <= opt && opt <= "z") { + skip = true + break + } + } + if !skip && (len(arg) > 2 || !ok) { + shortopts = arg[1:] + goto L + } + } + if !ok { + if positionalVal.IsValid() && len(rest) > 0 { + positionalVal.Set(reflect.Append(positionalVal, reflect.ValueOf(arg))) + } else { + rest = append(rest, arg) + } + continue + } + S: + switch val.Kind() { + case reflect.Bool: + val.SetBool(true) + case reflect.String: + if i++; i >= len(args) { + return nil, fmt.Errorf("expected argument for flag `%s'", arg) + } + val.SetString(args[i]) + case reflect.Ptr: + if val.Type().Elem().Kind() == reflect.Int { + if i++; i >= len(args) { + return nil, fmt.Errorf("expected argument for flag `%s'", arg) + } + v, err := strconv.Atoi(args[i]) + if err != nil { + return nil, fmt.Errorf("invalid argument for flag `%s': %w", arg, err) + } + val.Set(reflect.New(val.Type().Elem())) + val.Elem().SetInt(int64(v)) + } + case reflect.Slice: + if _, ok := longToPositional[arg[2:]]; ok { + if positionalVal.IsValid() { + for positionalVal.Len() > val.Len() { + val.Set(reflect.Append(val, reflect.Zero(val.Type().Elem()))) + } + } + positionalVal = val + } else { + if i++; i >= len(args) { + return nil, fmt.Errorf("expected argument for flag `%s'", arg) + } + val.Set(reflect.Append(val, reflect.ValueOf(args[i]))) + } + case reflect.Map: + if i += 2; i >= len(args) { + return nil, fmt.Errorf("expected 2 arguments for flag `%s'", arg) + } + if val.IsNil() { + val.Set(reflect.MakeMap(val.Type())) + } + name := args[i-1] + if _, ok := mapKeys[name]; !ok { + mapKeys[name] = struct{}{} + val.SetMapIndex(reflect.ValueOf(name), reflect.ValueOf(args[i])) + } + } + L: + if shortopts != "" { + opt := shortopts[:1] + if val, ok = shortToValue[opt]; !ok { + return nil, fmt.Errorf("unknown flag `%s'", opt) + } + if val.Kind() != reflect.Bool && len(shortopts) > 1 { + if shortopts[1] == '=' { + args[i] = shortopts[2:] + } else { + args[i] = shortopts[1:] + } + i-- + shortopts = "" + } else { + shortopts = shortopts[1:] + } + arg = "-" + opt + goto S + } + } + return rest, nil +} + +func formatFlags(opts any) string { + val := reflect.ValueOf(opts).Elem() + typ := val.Type() + var sb strings.Builder + sb.WriteString("Command Options:\n") + for i, l := 0, typ.NumField(); i < l; i++ { + tag := typ.Field(i).Tag + if i == l-1 { + sb.WriteString("\nHelp Option:\n") + } + sb.WriteString(" ") + var short bool + if flag, ok := tag.Lookup("short"); ok { + sb.WriteString("-") + sb.WriteString(flag) + short = true + } else { + sb.WriteString(" ") + } + m := sb.Len() + if flag, ok := tag.Lookup("long"); ok { + if short { + sb.WriteString(", ") + } else { + sb.WriteString(" ") + } + sb.WriteString("--") + sb.WriteString(flag) + switch val.Field(i).Kind() { + case reflect.Bool: + sb.WriteString(" ") + case reflect.Map: + if strings.HasSuffix(flag, "file") { + sb.WriteString(" name file") + } else { + sb.WriteString(" name value") + } + default: + if _, ok = tag.Lookup("positional"); !ok { + sb.WriteString("=") + } + } + } else { + sb.WriteString("=") + } + sb.WriteString(" "[:24-sb.Len()+m]) + sb.WriteString(tag.Get("description")) + sb.WriteString("\n") + } + return sb.String() +} diff --git a/gojq-extended/go.mod b/gojq-extended/go.mod new file mode 100644 index 0000000..1f5819a --- /dev/null +++ b/gojq-extended/go.mod @@ -0,0 +1,15 @@ +module forgejo.owo.monster/chaos/psychonaut_journal_cli/gojq-extended + +go 1.23.2 + +require ( + github.com/itchyny/gojq v0.12.16 + github.com/mattn/go-isatty v0.0.20 + github.com/mattn/go-runewidth v0.0.16 +) + +require ( + github.com/itchyny/timefmt-go v0.1.6 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/sys v0.20.0 // indirect +) diff --git a/gojq-extended/go.sum b/gojq-extended/go.sum new file mode 100644 index 0000000..c303e1c --- /dev/null +++ b/gojq-extended/go.sum @@ -0,0 +1,14 @@ +github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= +github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= +github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= +github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/gojq-extended/inputs.go b/gojq-extended/inputs.go new file mode 100644 index 0000000..64b2f82 --- /dev/null +++ b/gojq-extended/inputs.go @@ -0,0 +1,375 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "os" + "strings" + + "github.com/itchyny/gojq" +) + +type inputReader struct { + io.Reader + file *os.File + buf *bytes.Buffer +} + +func newInputReader(r io.Reader) *inputReader { + if r, ok := r.(*os.File); ok { + if _, err := r.Seek(0, io.SeekCurrent); err == nil { + return &inputReader{r, r, nil} + } + } + var buf bytes.Buffer // do not use strings.Builder because we need to Reset + return &inputReader{io.TeeReader(r, &buf), nil, &buf} +} + +func (ir *inputReader) getContents(offset *int64, line *int) string { + if buf := ir.buf; buf != nil { + return buf.String() + } + if current, err := ir.file.Seek(0, io.SeekCurrent); err == nil { + defer func() { ir.file.Seek(current, io.SeekStart) }() + } + ir.file.Seek(0, io.SeekStart) + const bufSize = 16 * 1024 + var buf bytes.Buffer // do not use strings.Builder because we need to Reset + if offset != nil && *offset > bufSize { + buf.Grow(bufSize) + for *offset > bufSize { + n, err := io.Copy(&buf, io.LimitReader(ir.file, bufSize)) + *offset -= int64(n) + *line += bytes.Count(buf.Bytes(), []byte{'\n'}) + buf.Reset() + if err != nil || n == 0 { + break + } + } + } + var r io.Reader + if offset == nil { + r = ir.file + } else { + r = io.LimitReader(ir.file, bufSize*2) + } + io.Copy(&buf, r) + return buf.String() +} + +type inputIter interface { + gojq.Iter + io.Closer + Name() string +} + +type jsonInputIter struct { + next func() (any, error) + ir *inputReader + fname string + offset int64 + line int + err error +} + +func newJSONInputIter(r io.Reader, fname string) inputIter { + ir := newInputReader(r) + dec := json.NewDecoder(ir) + dec.UseNumber() + next := func() (v any, err error) { err = dec.Decode(&v); return } + return &jsonInputIter{next: next, ir: ir, fname: fname} +} + +func (i *jsonInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + v, err := i.next() + if err != nil { + if err == io.EOF { + i.err = err + return nil, false + } + var offset *int64 + var line *int + if err, ok := err.(*json.SyntaxError); ok { + err.Offset -= i.offset + offset, line = &err.Offset, &i.line + } + i.err = &jsonParseError{i.fname, i.ir.getContents(offset, line), i.line, err} + return i.err, true + } + if buf := i.ir.buf; buf != nil && buf.Len() >= 16*1024 { + i.offset += int64(buf.Len()) + i.line += bytes.Count(buf.Bytes(), []byte{'\n'}) + buf.Reset() + } + return v, true +} + +func (i *jsonInputIter) Close() error { + i.err = io.EOF + return nil +} + +func (i *jsonInputIter) Name() string { + return i.fname +} + +func newStreamInputIter(r io.Reader, fname string) inputIter { + ir := newInputReader(r) + dec := json.NewDecoder(ir) + dec.UseNumber() + return &jsonInputIter{next: newJSONStream(dec).next, ir: ir, fname: fname} +} + +type nullInputIter struct { + err error +} + +func newNullInputIter() inputIter { + return &nullInputIter{} +} + +func (i *nullInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + i.err = io.EOF + return nil, true +} + +func (i *nullInputIter) Close() error { + i.err = io.EOF + return nil +} + +func (*nullInputIter) Name() string { + return "" +} + +type filesInputIter struct { + newIter func(io.Reader, string) inputIter + fnames []string + stdin io.Reader + iter inputIter + file io.Reader + err error +} + +func newFilesInputIter( + newIter func(io.Reader, string) inputIter, fnames []string, stdin io.Reader, +) inputIter { + return &filesInputIter{newIter: newIter, fnames: fnames, stdin: stdin} +} + +func (i *filesInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + for { + if i.file == nil { + if len(i.fnames) == 0 { + i.err = io.EOF + if i.iter != nil { + i.iter.Close() + i.iter = nil + } + return nil, false + } + fname := i.fnames[0] + i.fnames = i.fnames[1:] + if fname == "-" && i.stdin != nil { + i.file, fname = i.stdin, "" + } else { + file, err := os.Open(fname) + if err != nil { + return err, true + } + i.file = file + } + if i.iter != nil { + i.iter.Close() + } + i.iter = i.newIter(i.file, fname) + } + if v, ok := i.iter.Next(); ok { + return v, ok + } + if r, ok := i.file.(io.Closer); ok && i.file != i.stdin { + r.Close() + } + i.file = nil + } +} + +func (i *filesInputIter) Close() error { + if i.file != nil { + if r, ok := i.file.(io.Closer); ok && i.file != i.stdin { + r.Close() + } + i.file = nil + i.err = io.EOF + } + return nil +} + +func (i *filesInputIter) Name() string { + if i.iter != nil { + return i.iter.Name() + } + return "" +} + +type rawInputIter struct { + r *bufio.Reader + fname string + err error +} + +func newRawInputIter(r io.Reader, fname string) inputIter { + return &rawInputIter{r: bufio.NewReader(r), fname: fname} +} + +func (i *rawInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + line, err := i.r.ReadString('\n') + if err != nil { + i.err = err + if err != io.EOF { + return err, true + } + if line == "" { + return nil, false + } + } + return strings.TrimSuffix(line, "\n"), true +} + +func (i *rawInputIter) Close() error { + i.err = io.EOF + return nil +} + +func (i *rawInputIter) Name() string { + return i.fname +} + +type slurpInputIter struct { + iter inputIter + err error +} + +func newSlurpInputIter(iter inputIter) inputIter { + return &slurpInputIter{iter: iter} +} + +func (i *slurpInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + var vs []any + var v any + var ok bool + for { + v, ok = i.iter.Next() + if !ok { + i.err = io.EOF + return vs, true + } + if i.err, ok = v.(error); ok { + return i.err, true + } + vs = append(vs, v) + } +} + +func (i *slurpInputIter) Close() error { + if i.iter != nil { + i.iter.Close() + i.iter = nil + i.err = io.EOF + } + return nil +} + +func (i *slurpInputIter) Name() string { + return i.iter.Name() +} + +type readAllIter struct { + r io.Reader + fname string + err error +} + +func newReadAllIter(r io.Reader, fname string) inputIter { + return &readAllIter{r: r, fname: fname} +} + +func (i *readAllIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + i.err = io.EOF + cnt, err := io.ReadAll(i.r) + if err != nil { + return err, true + } + return string(cnt), true +} + +func (i *readAllIter) Close() error { + i.err = io.EOF + return nil +} + +func (i *readAllIter) Name() string { + return i.fname +} + +type slurpRawInputIter struct { + iter inputIter + err error +} + +func newSlurpRawInputIter(iter inputIter) inputIter { + return &slurpRawInputIter{iter: iter} +} + +func (i *slurpRawInputIter) Next() (any, bool) { + if i.err != nil { + return nil, false + } + var vs []string + var v any + var ok bool + for { + v, ok = i.iter.Next() + if !ok { + i.err = io.EOF + return strings.Join(vs, ""), true + } + if i.err, ok = v.(error); ok { + return i.err, true + } + vs = append(vs, v.(string)) + } +} + +func (i *slurpRawInputIter) Close() error { + if i.iter != nil { + i.iter.Close() + i.iter = nil + i.err = io.EOF + } + return nil +} + +func (i *slurpRawInputIter) Name() string { + return i.iter.Name() +} diff --git a/gojq-extended/marshaler.go b/gojq-extended/marshaler.go new file mode 100644 index 0000000..9a85bcf --- /dev/null +++ b/gojq-extended/marshaler.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "io" + "strings" +) + +type marshaler interface { + marshal(any, io.Writer) error +} + +type rawMarshaler struct { + m marshaler + checkNul bool +} + +func (m *rawMarshaler) marshal(v any, w io.Writer) error { + if s, ok := v.(string); ok { + if m.checkNul && strings.ContainsRune(s, '\x00') { + return fmt.Errorf("cannot output a string containing NUL character: %q", s) + } + _, err := w.Write([]byte(s)) + return err + } + return m.m.marshal(v, w) +} diff --git a/gojq-extended/run.go b/gojq-extended/run.go new file mode 100644 index 0000000..0e09557 --- /dev/null +++ b/gojq-extended/run.go @@ -0,0 +1,13 @@ +package main + +import "os" + +// Run gojq. +func main() { + cli := cli{ + inStream: os.Stdin, + outStream: os.Stdout, + errStream: os.Stderr, + } + os.Exit(cli.run(os.Args[1:])) +} diff --git a/gojq-extended/stream.go b/gojq-extended/stream.go new file mode 100644 index 0000000..09e436b --- /dev/null +++ b/gojq-extended/stream.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "io" +) + +type jsonStream struct { + dec *json.Decoder + path []any + states []int +} + +func newJSONStream(dec *json.Decoder) *jsonStream { + return &jsonStream{dec: dec, states: []int{jsonStateTopValue}, path: []any{}} +} + +const ( + jsonStateTopValue = iota + jsonStateArrayStart + jsonStateArrayValue + jsonStateArrayEnd + jsonStateArrayEmptyEnd + jsonStateObjectStart + jsonStateObjectKey + jsonStateObjectValue + jsonStateObjectEnd + jsonStateObjectEmptyEnd +) + +func (s *jsonStream) next() (any, error) { + switch s.states[len(s.states)-1] { + case jsonStateArrayEnd, jsonStateObjectEnd: + s.path = s.path[:len(s.path)-1] + fallthrough + case jsonStateArrayEmptyEnd, jsonStateObjectEmptyEnd: + s.states = s.states[:len(s.states)-1] + } + if s.dec.More() { + switch s.states[len(s.states)-1] { + case jsonStateArrayValue: + s.path[len(s.path)-1] = s.path[len(s.path)-1].(int) + 1 + case jsonStateObjectValue: + s.path = s.path[:len(s.path)-1] + } + } + for { + token, err := s.dec.Token() + if err != nil { + if err == io.EOF && s.states[len(s.states)-1] != jsonStateTopValue { + err = io.ErrUnexpectedEOF + } + return nil, err + } + if d, ok := token.(json.Delim); ok { + switch d { + case '[', '{': + switch s.states[len(s.states)-1] { + case jsonStateArrayStart: + s.states[len(s.states)-1] = jsonStateArrayValue + case jsonStateObjectKey: + s.states[len(s.states)-1] = jsonStateObjectValue + } + if d == '[' { + s.states = append(s.states, jsonStateArrayStart) + s.path = append(s.path, 0) + } else { + s.states = append(s.states, jsonStateObjectStart) + } + case ']': + if s.states[len(s.states)-1] == jsonStateArrayStart { + s.states[len(s.states)-1] = jsonStateArrayEmptyEnd + s.path = s.path[:len(s.path)-1] + return []any{s.copyPath(), []any{}}, nil + } + s.states[len(s.states)-1] = jsonStateArrayEnd + return []any{s.copyPath()}, nil + case '}': + if s.states[len(s.states)-1] == jsonStateObjectStart { + s.states[len(s.states)-1] = jsonStateObjectEmptyEnd + return []any{s.copyPath(), map[string]any{}}, nil + } + s.states[len(s.states)-1] = jsonStateObjectEnd + return []any{s.copyPath()}, nil + default: + panic(d) + } + } else { + switch s.states[len(s.states)-1] { + case jsonStateArrayStart: + s.states[len(s.states)-1] = jsonStateArrayValue + fallthrough + case jsonStateArrayValue: + return []any{s.copyPath(), token}, nil + case jsonStateObjectStart, jsonStateObjectValue: + s.states[len(s.states)-1] = jsonStateObjectKey + s.path = append(s.path, token) + case jsonStateObjectKey: + s.states[len(s.states)-1] = jsonStateObjectValue + return []any{s.copyPath(), token}, nil + default: + s.states[len(s.states)-1] = jsonStateTopValue + return []any{s.copyPath(), token}, nil + } + } + } +} + +func (s *jsonStream) copyPath() []any { + path := make([]any, len(s.path)) + copy(path, s.path) + return path +} diff --git a/run.sh b/run.sh index 090f198..60a4efc 100755 --- a/run.sh +++ b/run.sh @@ -2,15 +2,33 @@ set -eu -SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" -cd "$SCRIPT_DIR/tool" +WORKING_DIRECTORY=$(pwd) +TOOL_DIR="$(cd -- "$(dirname -- "$0")" && pwd)/tool" +cd "${WORKING_DIRECTORY}" JQ=${JQ:-jq} -export JQ_FLAVOR=${JQ_FLAVOR:-${JQ}} +export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"} run() { - ${JQ} -nr -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath ./dropins)/${JQ_FLAVOR}" \ - --slurpfile exportFile "${EXPORT_FILE:-export.json}" \ + if [ -d "${TOOL_DIR}/lib/stubs/${JQ_FLAVOR}" ]; then + STUBS_DIR="${TOOL_DIR}/lib/stubs/${JQ_FLAVOR}" + else + STUBS_DIR="${TOOL_DIR}/lib/stubs/jq" + fi + + FILES_ARGS=() + if [ "${EXPORT_FILE:-}" != "" ] && [ -f "${EXPORT_FILE}" ]; then + FILES_ARGS+=(--arg exportFileName "${EXPORT_FILE:-}") + if [ "${JQ_FLAVOR}" != "gojq-extended" ]; then + FILES_ARGS+=(--slurpfile exportFile "${EXPORT_FILE}") + fi + else + FILES_ARGS+=(--argjson exportFileName null) + FILES_ARGS+=(--argjson exportFile null) + fi + + ${JQ} -nr -L "$(realpath "${TOOL_DIR}")" -L "$(realpath "${TOOL_DIR}/lib")" -L "$(realpath "${STUBS_DIR}")" \ + "${FILES_ARGS[@]}" \ 'include "main"; main' \ --args -- "$@" } diff --git a/runTests.sh b/runTests.sh index ca2be32..b0eb273 100755 --- a/runTests.sh +++ b/runTests.sh @@ -3,14 +3,21 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" cd "${SCRIPT_DIR}/tool" || return +JQ=${JQ:-jq} +export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"} export JQ_TYPECHECKING=1 +if [ ! -d "./lib/stubs/${JQ_FLAVOR}" ]; then + STUBS_DIR="./lib/stubs/${JQ_FLAVOR}" +else + STUBS_DIR="./lib/stubs/jq" +fi + runTests() { - JQ=${JQ:-jq} - export JQ_FLAVOR=${JQ_FLAVOR:-jq} echo "Running Tests with JQ=${JQ}" + export JQ_FLAVOR=${JQ_FLAVOR:-"$(basename "${JQ}")"} ${JQ} -nr \ - -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath ./dropins)/${JQ_FLAVOR}" \ + -L "$(realpath .)" -L "$(realpath ./lib)" -L "$(realpath "${STUBS_DIR}")" \ "include \"tests\"; testsMain" } diff --git a/tool/dropins/jq/dropins.jq b/tool/dropins/jq/dropins.jq deleted file mode 100644 index e69de29..0000000 diff --git a/tool/journalUtils.jq b/tool/journalUtils.jq index 2eb0fc2..7c0360b 100644 --- a/tool/journalUtils.jq +++ b/tool/journalUtils.jq @@ -1,5 +1,3 @@ -include "dropins"; - import "lib/utilsLib" as utilsLib; import "lib/stringLib" as stringLib; import "lib/numberLib" as numberLib; @@ -11,7 +9,7 @@ def formatExperienceTitle: def formatDose($dose; $unit; $isEstimate; $standardDeviation): if $dose == null then - "Unknown" + "Unknown \($unit)" else (if $isEstimate then "~" else "" end) as $estimate | (if $standardDeviation != null then "±\($standardDeviation)" else "" end) as $standardDeviation | @@ -34,7 +32,8 @@ def formatIngestionDose($customUnits): " (" + formatDose($customUnit.dose; $unit; $customUnit.isEstimate; $customUnit.estimatedDoseStandardDeviation) + " * " + - formatDose($ingestion.dose; $customUnit.unit; $ingestion.isDoseAnEstimate; $ingestion.estimatedDoseStandardDeviation) + formatDose($ingestion.dose; $customUnit.unit; $ingestion.isDoseAnEstimate; $ingestion.estimatedDoseStandardDeviation) + + ")" end; def formatIngestionTime: @@ -68,15 +67,4 @@ def formatIngestionROA($customUnits): formatIngestionROA($customUnits; {}); def formatIngestionInfo: . as $ingestionInfo | - if $ingestionInfo.dose == null then - "Unknown \($ingestionInfo.unit)" - else - if $ingestionInfo.isEstimate then "~" else "" end + - "\($ingestionInfo.dose * 100 | round / 100)" + - if $ingestionInfo.standardDeviation != null then - "±\($ingestionInfo.standardDeviation)" - else "" end + - - if $ingestionInfo.isUnknown then "+ Unknown" else "" end + - " \($ingestionInfo.unit)" - end; \ No newline at end of file + formatDose(.dose; .unit; .isEstimate; .standardDeviation); \ No newline at end of file diff --git a/tool/lib/.jq-lsp.jq b/tool/lib/.jq-lsp.jq new file mode 100644 index 0000000..a8fb776 --- /dev/null +++ b/tool/lib/.jq-lsp.jq @@ -0,0 +1,2 @@ +def _readFile($filename): empty; +def _writeFileString($filename): empty; \ No newline at end of file diff --git a/tool/lib/gojqExtendedLib.jq b/tool/lib/gojqExtendedLib.jq new file mode 100644 index 0000000..2b43f3f --- /dev/null +++ b/tool/lib/gojqExtendedLib.jq @@ -0,0 +1,7 @@ +include "stubs"; + +def checkSupported: + $ENV.JQ_FLAVOR == "gojq-extended"; + +def readFile($filename): _readFile($filename); +def writeFileString($filename): _writeFileString($filename); \ No newline at end of file diff --git a/tool/lib/journalLib.jq b/tool/lib/journalLib.jq index c46aa90..2dc3965 100644 --- a/tool/lib/journalLib.jq +++ b/tool/lib/journalLib.jq @@ -96,9 +96,7 @@ def ingestionsByConsumer: $consumerNames[] as $consumerName | { key: $consumerName, - value: $ingestions | map(select( - . | ingestionConsumerName == $consumerName - )), + value: $ingestions | map(select((. | ingestionConsumerName) == $consumerName)), } ] | from_entries; @@ -167,7 +165,7 @@ def experienceStats($customUnits): $ingestion | .substanceName as $name | .administrationRoute as $administrationRoute | - . | ingestionConsumerName as $consumerName | + (. | ingestionConsumerName) as $consumerName | $stats | .[$consumerName].[$ingestion.substanceName].[$administrationRoute] as $ingestionStats | diff --git a/tool/lib/journalLibTests.jq b/tool/lib/journalLibTests.jq index 92ffb46..1784390 100644 --- a/tool/lib/journalLibTests.jq +++ b/tool/lib/journalLibTests.jq @@ -1,5 +1,3 @@ -include "dropins"; - import "testLib" as testLib; import "journalLib" as journalLib; diff --git a/tool/lib/journalTypes.jq b/tool/lib/journalTypes.jq index 6e1602e..6d877a6 100644 --- a/tool/lib/journalTypes.jq +++ b/tool/lib/journalTypes.jq @@ -1,5 +1,3 @@ -include "dropins"; - import "typeLib" as typeLib; def ensureAdministrationRoute: diff --git a/tool/dropins/gojq/dropins.jq b/tool/lib/stubs/gojq-extended/stubs.jq similarity index 100% rename from tool/dropins/gojq/dropins.jq rename to tool/lib/stubs/gojq-extended/stubs.jq diff --git a/tool/lib/stubs/jq/stubs.jq b/tool/lib/stubs/jq/stubs.jq new file mode 100644 index 0000000..b362c74 --- /dev/null +++ b/tool/lib/stubs/jq/stubs.jq @@ -0,0 +1,2 @@ +def _readFile($filename): error("_readFile only supported in gojq-extended"); +def _writeFileString($filename): error("_writeFileString only supported in gojq-extended"); \ No newline at end of file diff --git a/tool/lib/utilsLib.jq b/tool/lib/utilsLib.jq index 79a5612..9dabdc3 100644 --- a/tool/lib/utilsLib.jq +++ b/tool/lib/utilsLib.jq @@ -1,5 +1,3 @@ -include "dropins"; - import "stringLib" as stringLib; def debugLog($target; $value): diff --git a/tool/main.jq b/tool/main.jq index fd2c332..8a6cb1a 100644 --- a/tool/main.jq +++ b/tool/main.jq @@ -1,11 +1,10 @@ -include "dropins"; - import "lib/typeLib" as typeLib; import "lib/argsLib" as argsLib; import "lib/stringLib" as stringLib; import "lib/tableLib" as tableLib; import "lib/utilsLib" as utilsLib; import "lib/journalLib" as journalLib; +import "lib/gojqExtendedLib" as gojqExtendedLib; import "lib/journalTypes" as journalTypes; import "journalUtils" as journalUtils; @@ -22,13 +21,14 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle): ($ingestions | journalLib::ingestionsSubstanceNames) as $substanceNames | ($ingestions | journalLib::ingestionsByConsumer) as $ingestionsByConsumer | + ($ingestionsByConsumer | keys) as $consumerNames | + "" as $experienceStatsText | $experienceStatsText | if $withTitle then . += ($experience | journalUtils::formatExperienceTitle | . + "\n") end | . as $experienceStatsText | - ($ingestionsByConsumer | keys) as $consumerNames | reduce $consumerNames[] as $consumerName ($experienceStatsText; . as $experienceStatsText | $experienceStatsText | @@ -38,7 +38,7 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle): ($ingestionsByConsumer[$consumerName] | journalLib::ingestionsSubstanceNames) as $consumerSubstanceNames | - $experienceStatsText | reduce $substanceNames[] as $substanceName (.; + $experienceStatsText | reduce $consumerSubstanceNames[] as $substanceName (.; . as $experienceStatsText | ($stats.[$consumerName].[$substanceName] | keys) as $ingestionMethods | @@ -52,8 +52,22 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle): .key as $ingestionMethod | .value as $ingestionInfo | + def formatIngestionInfo: + if $ingestionInfo.dose == null then + "Unknown \($ingestionInfo.unit)" + else + if $ingestionInfo.isEstimate then "~" else "" end + + "\($ingestionInfo.dose * 100 | round / 100)" + + if $ingestionInfo.standardDeviation != null then + "±\($ingestionInfo.standardDeviation)" + else "" end + + + if $ingestionInfo.isUnknown then "+ Unknown" else "" end + + " \($ingestionInfo.unit)" + end; + $experienceStatsText | - . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($ingestionInfo | journalUtils::formatIngestionInfo)\n" | + . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($ingestionInfo | formatIngestionInfo)\n" | . as $experienceStatsText | $experienceStatsText @@ -133,14 +147,47 @@ def main: "" ] | join("\n") | halt_error(1); - $ARGS.named["exportFile"][0] as $exportData | - $exportData | journalTypes::ensureExportData | - ($ARGS | argsLib::parseArgs) as $parsedArgs | $parsedArgs.nonArgs[0] as $program | ($parsedArgs | .nonArgs |= $parsedArgs.nonArgs[1:]) as $parsedArgs | + if gojqExtendedLib::checkSupported then + if $parsedArgs.longArgs | has("export-file") then + $parsedArgs.longArgs["export-file"] + else + $ARGS.named["exportFileName"] + end | . as $exportFileName | + { + name: $exportFileName, + content: gojqExtendedLib::readFile($exportFileName) | fromjson + } + else + if $parsedArgs.longArgs | has("export-file") then + debug("--export-file was provided but this version of jq doesn't support reading files" + + "using EXPORT_FILE instead") + end | + { + name: $ARGS.named["exportFileName"], + content: $ARGS.named["exportFile"][0] + } + end | . as $exportFile | + + $exportFile.name as $exportFileName | + $exportFile.content as $exportData | + + if $exportData == null then + if gojqExtendedLib::checkSupported then + "please set EXPORT_FILE= or --export-file to a valid psychonaut journal export file\n" | + halt_error(1) + else + "please set EXPORT_FILE= to a valid psychonaut journal export file\n" | + halt_error(1) + end + end | + + $exportData | journalTypes::ensureExportData | + if $program == null then if any($parsedArgs.shortArgs[]; . == "h") then usage end | if ($parsedArgs.longArgs | has("help")) then usage end | diff --git a/tool/tests.jq b/tool/tests.jq index c2726de..fd66db2 100644 --- a/tool/tests.jq +++ b/tool/tests.jq @@ -1,5 +1,3 @@ -include "dropins"; - import "lib/testLib" as testLib; import "lib/typeLib" as typeLib; import "lib/journalLibTests" as journalLibTests;