vendor tsgo

This commit is contained in:
2026-07-09 16:50:43 -04:00
parent c06ea2e5a4
commit 98978e4930
5804 changed files with 1556156 additions and 101 deletions

View File

@@ -0,0 +1 @@
* text=auto eol=lf

View File

@@ -0,0 +1,10 @@
.vscode/
*.exe
# testing
testdata
# go workspaces
go.work
go.work.sum

View File

@@ -0,0 +1,147 @@
linters:
enable:
# style
- containedctx # struct contains a context
- dupl # duplicate code
- errname # erorrs are named correctly
- nolintlint # "//nolint" directives are properly explained
- revive # golint replacement
- unconvert # unnecessary conversions
- wastedassign
# bugs, performance, unused, etc ...
- contextcheck # function uses a non-inherited context
- errorlint # errors not wrapped for 1.13
- exhaustive # check exhaustiveness of enum switch statements
- gofmt # files are gofmt'ed
- gosec # security
- nilerr # returns nil even with non-nil error
- thelper # test helpers without t.Helper()
- unparam # unused function params
issues:
exclude-dirs:
- pkg/etw/sample
exclude-rules:
# err is very often shadowed in nested scopes
- linters:
- govet
text: '^shadow: declaration of "err" shadows declaration'
# ignore long lines for skip autogen directives
- linters:
- revive
text: "^line-length-limit: "
source: "^//(go:generate|sys) "
#TODO: remove after upgrading to go1.18
# ignore comment spacing for nolint and sys directives
- linters:
- revive
text: "^comment-spacings: no space between comment delimiter and comment text"
source: "//(cspell:|nolint:|sys |todo)"
# not on go 1.18 yet, so no any
- linters:
- revive
text: "^use-any: since GO 1.18 'interface{}' can be replaced by 'any'"
# allow unjustified ignores of error checks in defer statements
- linters:
- nolintlint
text: "^directive `//nolint:errcheck` should provide explanation"
source: '^\s*defer '
# allow unjustified ignores of error lints for io.EOF
- linters:
- nolintlint
text: "^directive `//nolint:errorlint` should provide explanation"
source: '[=|!]= io.EOF'
linters-settings:
exhaustive:
default-signifies-exhaustive: true
govet:
enable-all: true
disable:
# struct order is often for Win32 compat
# also, ignore pointer bytes/GC issues for now until performance becomes an issue
- fieldalignment
nolintlint:
require-explanation: true
require-specific: true
revive:
# revive is more configurable than static check, so likely the preferred alternative to static-check
# (once the perf issue is solved: https://github.com/golangci/golangci-lint/issues/2997)
enable-all-rules:
true
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
rules:
# rules with required arguments
- name: argument-limit
disabled: true
- name: banned-characters
disabled: true
- name: cognitive-complexity
disabled: true
- name: cyclomatic
disabled: true
- name: file-header
disabled: true
- name: function-length
disabled: true
- name: function-result-limit
disabled: true
- name: max-public-structs
disabled: true
# geneally annoying rules
- name: add-constant # complains about any and all strings and integers
disabled: true
- name: confusing-naming # we frequently use "Foo()" and "foo()" together
disabled: true
- name: flag-parameter # excessive, and a common idiom we use
disabled: true
- name: unhandled-error # warns over common fmt.Print* and io.Close; rely on errcheck instead
disabled: true
# general config
- name: line-length-limit
arguments:
- 140
- name: var-naming
arguments:
- []
- - CID
- CRI
- CTRD
- DACL
- DLL
- DOS
- ETW
- FSCTL
- GCS
- GMSA
- HCS
- HV
- IO
- LCOW
- LDAP
- LPAC
- LTSC
- MMIO
- NT
- OCI
- PMEM
- PWSH
- RX
- SACl
- SID
- SMB
- TX
- VHD
- VHDX
- VMID
- VPCI
- WCOW
- WIM

View File

@@ -0,0 +1 @@
* @microsoft/containerplat

View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Microsoft
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -0,0 +1,89 @@
# go-winio [![Build Status](https://github.com/microsoft/go-winio/actions/workflows/ci.yml/badge.svg)](https://github.com/microsoft/go-winio/actions/workflows/ci.yml)
This repository contains utilities for efficiently performing Win32 IO operations in
Go. Currently, this is focused on accessing named pipes and other file handles, and
for using named pipes as a net transport.
This code relies on IO completion ports to avoid blocking IO on system threads, allowing Go
to reuse the thread to schedule another goroutine. This limits support to Windows Vista and
newer operating systems. This is similar to the implementation of network sockets in Go's net
package.
Please see the LICENSE file for licensing information.
## Contributing
This project welcomes contributions and suggestions.
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that
you have the right to, and actually do, grant us the rights to use your contribution.
For details, visit [Microsoft CLA](https://cla.microsoft.com).
When you submit a pull request, a CLA-bot will automatically determine whether you need to
provide a CLA and decorate the PR appropriately (e.g., label, comment).
Simply follow the instructions provided by the bot.
You will only need to do this once across all repos using our CLA.
Additionally, the pull request pipeline requires the following steps to be performed before
mergining.
### Code Sign-Off
We require that contributors sign their commits using [`git commit --signoff`][git-commit-s]
to certify they either authored the work themselves or otherwise have permission to use it in this project.
A range of commits can be signed off using [`git rebase --signoff`][git-rebase-s].
Please see [the developer certificate](https://developercertificate.org) for more info,
as well as to make sure that you can attest to the rules listed.
Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
### Linting
Code must pass a linting stage, which uses [`golangci-lint`][lint].
The linting settings are stored in [`.golangci.yaml`](./.golangci.yaml), and can be run
automatically with VSCode by adding the following to your workspace or folder settings:
```json
"go.lintTool": "golangci-lint",
"go.lintOnSave": "package",
```
Additional editor [integrations options are also available][lint-ide].
Alternatively, `golangci-lint` can be [installed locally][lint-install] and run from the repo root:
```shell
# use . or specify a path to only lint a package
# to show all lint errors, use flags "--max-issues-per-linter=0 --max-same-issues=0"
> golangci-lint run ./...
```
### Go Generate
The pipeline checks that auto-generated code, via `go generate`, are up to date.
This can be done for the entire repo:
```shell
> go generate ./...
```
## Code of Conduct
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
## Special Thanks
Thanks to [natefinch][natefinch] for the inspiration for this library.
See [npipe](https://github.com/natefinch/npipe) for another named pipe implementation.
[lint]: https://golangci-lint.run/
[lint-ide]: https://golangci-lint.run/usage/integrations/#editor-integration
[lint-install]: https://golangci-lint.run/usage/install/#local-installation
[git-commit-s]: https://git-scm.com/docs/git-commit#Documentation/git-commit.txt--s
[git-rebase-s]: https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---signoff
[natefinch]: https://github.com/natefinch

View File

@@ -0,0 +1,41 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->

View File

@@ -0,0 +1,287 @@
//go:build windows
// +build windows
package winio
import (
"encoding/binary"
"errors"
"fmt"
"io"
"os"
"runtime"
"unicode/utf16"
"github.com/Microsoft/go-winio/internal/fs"
"golang.org/x/sys/windows"
)
//sys backupRead(h windows.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
//sys backupWrite(h windows.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite
const (
BackupData = uint32(iota + 1)
BackupEaData
BackupSecurity
BackupAlternateData
BackupLink
BackupPropertyData
BackupObjectId //revive:disable-line:var-naming ID, not Id
BackupReparseData
BackupSparseBlock
BackupTxfsData
)
const (
StreamSparseAttributes = uint32(8)
)
//nolint:revive // var-naming: ALL_CAPS
const (
WRITE_DAC = windows.WRITE_DAC
WRITE_OWNER = windows.WRITE_OWNER
ACCESS_SYSTEM_SECURITY = windows.ACCESS_SYSTEM_SECURITY
)
// BackupHeader represents a backup stream of a file.
type BackupHeader struct {
//revive:disable-next-line:var-naming ID, not Id
Id uint32 // The backup stream ID
Attributes uint32 // Stream attributes
Size int64 // The size of the stream in bytes
Name string // The name of the stream (for BackupAlternateData only).
Offset int64 // The offset of the stream in the file (for BackupSparseBlock only).
}
type win32StreamID struct {
StreamID uint32
Attributes uint32
Size uint64
NameSize uint32
}
// BackupStreamReader reads from a stream produced by the BackupRead Win32 API and produces a series
// of BackupHeader values.
type BackupStreamReader struct {
r io.Reader
bytesLeft int64
}
// NewBackupStreamReader produces a BackupStreamReader from any io.Reader.
func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
return &BackupStreamReader{r, 0}
}
// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
// it was not completely read.
func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if r.bytesLeft > 0 { //nolint:nestif // todo: flatten this
if s, ok := r.r.(io.Seeker); ok {
// Make sure Seek on io.SeekCurrent sometimes succeeds
// before trying the actual seek.
if _, err := s.Seek(0, io.SeekCurrent); err == nil {
if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil {
return nil, err
}
r.bytesLeft = 0
}
}
if _, err := io.Copy(io.Discard, r); err != nil {
return nil, err
}
}
var wsi win32StreamID
if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
return nil, err
}
hdr := &BackupHeader{
Id: wsi.StreamID,
Attributes: wsi.Attributes,
Size: int64(wsi.Size),
}
if wsi.NameSize != 0 {
name := make([]uint16, int(wsi.NameSize/2))
if err := binary.Read(r.r, binary.LittleEndian, name); err != nil {
return nil, err
}
hdr.Name = windows.UTF16ToString(name)
}
if wsi.StreamID == BackupSparseBlock {
if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
return nil, err
}
hdr.Size -= 8
}
r.bytesLeft = hdr.Size
return hdr, nil
}
// Read reads from the current backup stream.
func (r *BackupStreamReader) Read(b []byte) (int, error) {
if r.bytesLeft == 0 {
return 0, io.EOF
}
if int64(len(b)) > r.bytesLeft {
b = b[:r.bytesLeft]
}
n, err := r.r.Read(b)
r.bytesLeft -= int64(n)
if err == io.EOF {
err = io.ErrUnexpectedEOF
} else if r.bytesLeft == 0 && err == nil {
err = io.EOF
}
return n, err
}
// BackupStreamWriter writes a stream compatible with the BackupWrite Win32 API.
type BackupStreamWriter struct {
w io.Writer
bytesLeft int64
}
// NewBackupStreamWriter produces a BackupStreamWriter on top of an io.Writer.
func NewBackupStreamWriter(w io.Writer) *BackupStreamWriter {
return &BackupStreamWriter{w, 0}
}
// WriteHeader writes the next backup stream header and prepares for calls to Write().
func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
if w.bytesLeft != 0 {
return fmt.Errorf("missing %d bytes", w.bytesLeft)
}
name := utf16.Encode([]rune(hdr.Name))
wsi := win32StreamID{
StreamID: hdr.Id,
Attributes: hdr.Attributes,
Size: uint64(hdr.Size),
NameSize: uint32(len(name) * 2),
}
if hdr.Id == BackupSparseBlock {
// Include space for the int64 block offset
wsi.Size += 8
}
if err := binary.Write(w.w, binary.LittleEndian, &wsi); err != nil {
return err
}
if len(name) != 0 {
if err := binary.Write(w.w, binary.LittleEndian, name); err != nil {
return err
}
}
if hdr.Id == BackupSparseBlock {
if err := binary.Write(w.w, binary.LittleEndian, hdr.Offset); err != nil {
return err
}
}
w.bytesLeft = hdr.Size
return nil
}
// Write writes to the current backup stream.
func (w *BackupStreamWriter) Write(b []byte) (int, error) {
if w.bytesLeft < int64(len(b)) {
return 0, fmt.Errorf("too many bytes by %d", int64(len(b))-w.bytesLeft)
}
n, err := w.w.Write(b)
w.bytesLeft -= int64(n)
return n, err
}
// BackupFileReader provides an io.ReadCloser interface on top of the BackupRead Win32 API.
type BackupFileReader struct {
f *os.File
includeSecurity bool
ctx uintptr
}
// NewBackupFileReader returns a new BackupFileReader from a file handle. If includeSecurity is true,
// Read will attempt to read the security descriptor of the file.
func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader {
r := &BackupFileReader{f, includeSecurity, 0}
return r
}
// Read reads a backup stream from the file by calling the Win32 API BackupRead().
func (r *BackupFileReader) Read(b []byte) (int, error) {
var bytesRead uint32
err := backupRead(windows.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
if err != nil {
return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err}
}
runtime.KeepAlive(r.f)
if bytesRead == 0 {
return 0, io.EOF
}
return int(bytesRead), nil
}
// Close frees Win32 resources associated with the BackupFileReader. It does not close
// the underlying file.
func (r *BackupFileReader) Close() error {
if r.ctx != 0 {
_ = backupRead(windows.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
runtime.KeepAlive(r.f)
r.ctx = 0
}
return nil
}
// BackupFileWriter provides an io.WriteCloser interface on top of the BackupWrite Win32 API.
type BackupFileWriter struct {
f *os.File
includeSecurity bool
ctx uintptr
}
// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true,
// Write() will attempt to restore the security descriptor from the stream.
func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter {
w := &BackupFileWriter{f, includeSecurity, 0}
return w
}
// Write restores a portion of the file using the provided backup stream.
func (w *BackupFileWriter) Write(b []byte) (int, error) {
var bytesWritten uint32
err := backupWrite(windows.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
if err != nil {
return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err}
}
runtime.KeepAlive(w.f)
if int(bytesWritten) != len(b) {
return int(bytesWritten), errors.New("not all bytes could be written")
}
return len(b), nil
}
// Close frees Win32 resources associated with the BackupFileWriter. It does not
// close the underlying file.
func (w *BackupFileWriter) Close() error {
if w.ctx != 0 {
_ = backupWrite(windows.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
runtime.KeepAlive(w.f)
w.ctx = 0
}
return nil
}
// OpenForBackup opens a file or directory, potentially skipping access checks if the backup
// or restore privileges have been acquired.
//
// If the file opened was a directory, it cannot be used with Readdir().
func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
h, err := fs.CreateFile(path,
fs.AccessMask(access),
fs.FileShareMode(share),
nil,
fs.FileCreationDisposition(createmode),
fs.FILE_FLAG_BACKUP_SEMANTICS|fs.FILE_FLAG_OPEN_REPARSE_POINT,
0,
)
if err != nil {
err = &os.PathError{Op: "open", Path: path, Err: err}
return nil, err
}
return os.NewFile(uintptr(h), path), nil
}

22
tools/tsgo/vendor/github.com/Microsoft/go-winio/doc.go generated vendored Normal file
View File

@@ -0,0 +1,22 @@
// This package provides utilities for efficiently performing Win32 IO operations in Go.
// Currently, this package is provides support for genreal IO and management of
// - named pipes
// - files
// - [Hyper-V sockets]
//
// This code is similar to Go's [net] package, and uses IO completion ports to avoid
// blocking IO on system threads, allowing Go to reuse the thread to schedule other goroutines.
//
// This limits support to Windows Vista and newer operating systems.
//
// Additionally, this package provides support for:
// - creating and managing GUIDs
// - writing to [ETW]
// - opening and manageing VHDs
// - parsing [Windows Image files]
// - auto-generating Win32 API code
//
// [Hyper-V sockets]: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service
// [ETW]: https://docs.microsoft.com/en-us/windows-hardware/drivers/devtest/event-tracing-for-windows--etw-
// [Windows Image files]: https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/work-with-windows-images
package winio

137
tools/tsgo/vendor/github.com/Microsoft/go-winio/ea.go generated vendored Normal file
View File

@@ -0,0 +1,137 @@
package winio
import (
"bytes"
"encoding/binary"
"errors"
)
type fileFullEaInformation struct {
NextEntryOffset uint32
Flags uint8
NameLength uint8
ValueLength uint16
}
var (
fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
errEaNameTooLarge = errors.New("extended attribute name too large")
errEaValueTooLarge = errors.New("extended attribute value too large")
)
// ExtendedAttribute represents a single Windows EA.
type ExtendedAttribute struct {
Name string
Value []byte
Flags uint8
}
func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
var info fileFullEaInformation
err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
if err != nil {
err = errInvalidEaBuffer
return ea, nb, err
}
nameOffset := fileFullEaInformationSize
nameLen := int(info.NameLength)
valueOffset := nameOffset + int(info.NameLength) + 1
valueLen := int(info.ValueLength)
nextOffset := int(info.NextEntryOffset)
if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
err = errInvalidEaBuffer
return ea, nb, err
}
ea.Name = string(b[nameOffset : nameOffset+nameLen])
ea.Value = b[valueOffset : valueOffset+valueLen]
ea.Flags = info.Flags
if info.NextEntryOffset != 0 {
nb = b[info.NextEntryOffset:]
}
return ea, nb, err
}
// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
for len(b) != 0 {
ea, nb, err := parseEa(b)
if err != nil {
return nil, err
}
eas = append(eas, ea)
b = nb
}
return eas, err
}
func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
if int(uint8(len(ea.Name))) != len(ea.Name) {
return errEaNameTooLarge
}
if int(uint16(len(ea.Value))) != len(ea.Value) {
return errEaValueTooLarge
}
entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
withPadding := (entrySize + 3) &^ 3
nextOffset := uint32(0)
if !last {
nextOffset = withPadding
}
info := fileFullEaInformation{
NextEntryOffset: nextOffset,
Flags: ea.Flags,
NameLength: uint8(len(ea.Name)),
ValueLength: uint16(len(ea.Value)),
}
err := binary.Write(buf, binary.LittleEndian, &info)
if err != nil {
return err
}
_, err = buf.Write([]byte(ea.Name))
if err != nil {
return err
}
err = buf.WriteByte(0)
if err != nil {
return err
}
_, err = buf.Write(ea.Value)
if err != nil {
return err
}
_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
if err != nil {
return err
}
return nil
}
// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
// buffer for use with BackupWrite, ZwSetEaFile, etc.
func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
var buf bytes.Buffer
for i := range eas {
last := false
if i == len(eas)-1 {
last = true
}
err := writeEa(&buf, &eas[i], last)
if err != nil {
return nil, err
}
}
return buf.Bytes(), nil
}

320
tools/tsgo/vendor/github.com/Microsoft/go-winio/file.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
//go:build windows
// +build windows
package winio
import (
"errors"
"io"
"runtime"
"sync"
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/windows"
)
//sys cancelIoEx(file windows.Handle, o *windows.Overlapped) (err error) = CancelIoEx
//sys createIoCompletionPort(file windows.Handle, port windows.Handle, key uintptr, threadCount uint32) (newport windows.Handle, err error) = CreateIoCompletionPort
//sys getQueuedCompletionStatus(port windows.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
//sys setFileCompletionNotificationModes(h windows.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
//sys wsaGetOverlappedResult(h windows.Handle, o *windows.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult
var (
ErrFileClosed = errors.New("file has already been closed")
ErrTimeout = &timeoutError{}
)
type timeoutError struct{}
func (*timeoutError) Error() string { return "i/o timeout" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return true }
type timeoutChan chan struct{}
var ioInitOnce sync.Once
var ioCompletionPort windows.Handle
// ioResult contains the result of an asynchronous IO operation.
type ioResult struct {
bytes uint32
err error
}
// ioOperation represents an outstanding asynchronous Win32 IO.
type ioOperation struct {
o windows.Overlapped
ch chan ioResult
}
func initIO() {
h, err := createIoCompletionPort(windows.InvalidHandle, 0, 0, 0xffffffff)
if err != nil {
panic(err)
}
ioCompletionPort = h
go ioCompletionProcessor(h)
}
// win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
// It takes ownership of this handle and will close it if it is garbage collected.
type win32File struct {
handle windows.Handle
wg sync.WaitGroup
wgLock sync.RWMutex
closing atomic.Bool
socket bool
readDeadline deadlineHandler
writeDeadline deadlineHandler
}
type deadlineHandler struct {
setLock sync.Mutex
channel timeoutChan
channelLock sync.RWMutex
timer *time.Timer
timedout atomic.Bool
}
// makeWin32File makes a new win32File from an existing file handle.
func makeWin32File(h windows.Handle) (*win32File, error) {
f := &win32File{handle: h}
ioInitOnce.Do(initIO)
_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
if err != nil {
return nil, err
}
err = setFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
if err != nil {
return nil, err
}
f.readDeadline.channel = make(timeoutChan)
f.writeDeadline.channel = make(timeoutChan)
return f, nil
}
// Deprecated: use NewOpenFile instead.
func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return NewOpenFile(windows.Handle(h))
}
func NewOpenFile(h windows.Handle) (io.ReadWriteCloser, error) {
// If we return the result of makeWin32File directly, it can result in an
// interface-wrapped nil, rather than a nil interface value.
f, err := makeWin32File(h)
if err != nil {
return nil, err
}
return f, nil
}
// closeHandle closes the resources associated with a Win32 handle.
func (f *win32File) closeHandle() {
f.wgLock.Lock()
// Atomically set that we are closing, releasing the resources only once.
if !f.closing.Swap(true) {
f.wgLock.Unlock()
// cancel all IO and wait for it to complete
_ = cancelIoEx(f.handle, nil)
f.wg.Wait()
// at this point, no new IO can start
windows.Close(f.handle)
f.handle = 0
} else {
f.wgLock.Unlock()
}
}
// Close closes a win32File.
func (f *win32File) Close() error {
f.closeHandle()
return nil
}
// IsClosed checks if the file has been closed.
func (f *win32File) IsClosed() bool {
return f.closing.Load()
}
// prepareIO prepares for a new IO operation.
// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
func (f *win32File) prepareIO() (*ioOperation, error) {
f.wgLock.RLock()
if f.closing.Load() {
f.wgLock.RUnlock()
return nil, ErrFileClosed
}
f.wg.Add(1)
f.wgLock.RUnlock()
c := &ioOperation{}
c.ch = make(chan ioResult)
return c, nil
}
// ioCompletionProcessor processes completed async IOs forever.
func ioCompletionProcessor(h windows.Handle) {
for {
var bytes uint32
var key uintptr
var op *ioOperation
err := getQueuedCompletionStatus(h, &bytes, &key, &op, windows.INFINITE)
if op == nil {
panic(err)
}
op.ch <- ioResult{bytes, err}
}
}
// todo: helsaawy - create an asyncIO version that takes a context
// asyncIO processes the return value from ReadFile or WriteFile, blocking until
// the operation has actually completed.
func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != windows.ERROR_IO_PENDING { //nolint:errorlint // err is Errno
return int(bytes), err
}
if f.closing.Load() {
_ = cancelIoEx(f.handle, &c.o)
}
var timeout timeoutChan
if d != nil {
d.channelLock.Lock()
timeout = d.channel
d.channelLock.Unlock()
}
var r ioResult
select {
case r = <-c.ch:
err = r.err
if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
if f.closing.Load() {
err = ErrFileClosed
}
} else if err != nil && f.socket {
// err is from Win32. Query the overlapped structure to get the winsock error.
var bytes, flags uint32
err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
}
case <-timeout:
_ = cancelIoEx(f.handle, &c.o)
r = <-c.ch
err = r.err
if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
err = ErrTimeout
}
}
// runtime.KeepAlive is needed, as c is passed via native
// code to ioCompletionProcessor, c must remain alive
// until the channel read is complete.
// todo: (de)allocate *ioOperation via win32 heap functions, instead of needing to KeepAlive?
runtime.KeepAlive(c)
return int(r.bytes), err
}
// Read reads from a file handle.
func (f *win32File) Read(b []byte) (int, error) {
c, err := f.prepareIO()
if err != nil {
return 0, err
}
defer f.wg.Done()
if f.readDeadline.timedout.Load() {
return 0, ErrTimeout
}
var bytes uint32
err = windows.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIO(c, &f.readDeadline, bytes, err)
runtime.KeepAlive(b)
// Handle EOF conditions.
if err == nil && n == 0 && len(b) != 0 {
return 0, io.EOF
} else if err == windows.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno
return 0, io.EOF
}
return n, err
}
// Write writes to a file handle.
func (f *win32File) Write(b []byte) (int, error) {
c, err := f.prepareIO()
if err != nil {
return 0, err
}
defer f.wg.Done()
if f.writeDeadline.timedout.Load() {
return 0, ErrTimeout
}
var bytes uint32
err = windows.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIO(c, &f.writeDeadline, bytes, err)
runtime.KeepAlive(b)
return n, err
}
func (f *win32File) SetReadDeadline(deadline time.Time) error {
return f.readDeadline.set(deadline)
}
func (f *win32File) SetWriteDeadline(deadline time.Time) error {
return f.writeDeadline.set(deadline)
}
func (f *win32File) Flush() error {
return windows.FlushFileBuffers(f.handle)
}
func (f *win32File) Fd() uintptr {
return uintptr(f.handle)
}
func (d *deadlineHandler) set(deadline time.Time) error {
d.setLock.Lock()
defer d.setLock.Unlock()
if d.timer != nil {
if !d.timer.Stop() {
<-d.channel
}
d.timer = nil
}
d.timedout.Store(false)
select {
case <-d.channel:
d.channelLock.Lock()
d.channel = make(chan struct{})
d.channelLock.Unlock()
default:
}
if deadline.IsZero() {
return nil
}
timeoutIO := func() {
d.timedout.Store(true)
close(d.channel)
}
now := time.Now()
duration := deadline.Sub(now)
if deadline.After(now) {
// Deadline is in the future, set a timer to wait
d.timer = time.AfterFunc(duration, timeoutIO)
} else {
// Deadline is in the past. Cancel all pending IO now.
timeoutIO()
}
return nil
}

View File

@@ -0,0 +1,106 @@
//go:build windows
// +build windows
package winio
import (
"os"
"runtime"
"unsafe"
"golang.org/x/sys/windows"
)
// FileBasicInfo contains file access time and file attributes information.
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
_ uint32 // padding
}
// alignedFileBasicInfo is a FileBasicInfo, but aligned to uint64 by containing
// uint64 rather than windows.Filetime. Filetime contains two uint32s. uint64
// alignment is necessary to pass this as FILE_BASIC_INFO.
type alignedFileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime uint64
FileAttributes uint32
_ uint32 // padding
}
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &alignedFileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
// Reinterpret the alignedFileBasicInfo as a FileBasicInfo so it matches the
// public API of this module. The data may be unnecessarily aligned.
return (*FileBasicInfo)(unsafe.Pointer(bi)), nil
}
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
// Create an alignedFileBasicInfo based on a FileBasicInfo. The copy is
// suitable to pass to GetFileInformationByHandleEx.
biAligned := *(*alignedFileBasicInfo)(unsafe.Pointer(bi))
if err := windows.SetFileInformationByHandle(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(&biAligned)),
uint32(unsafe.Sizeof(biAligned)),
); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return nil
}
// FileStandardInfo contains extended information for the file.
// FILE_STANDARD_INFO in WinBase.h
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info
type FileStandardInfo struct {
AllocationSize, EndOfFile int64
NumberOfLinks uint32
DeletePending, Directory bool
}
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()),
windows.FileStandardInfo,
(*byte)(unsafe.Pointer(si)),
uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return si, nil
}
// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
// unique on a system.
type FileIDInfo struct {
VolumeSerialNumber uint64
FileID [16]byte
}
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileIdInfo,
(*byte)(unsafe.Pointer(fileID)),
uint32(unsafe.Sizeof(*fileID)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return fileID, nil
}

View File

@@ -0,0 +1,582 @@
//go:build windows
// +build windows
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/socket"
"github.com/Microsoft/go-winio/pkg/guid"
)
const afHVSock = 34 // AF_HYPERV
// Well known Service and VM IDs
// https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards
// HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions.
func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000
return guid.GUID{}
}
// HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions.
func HvsockGUIDBroadcast() guid.GUID { // ffffffff-ffff-ffff-ffff-ffffffffffff
return guid.GUID{
Data1: 0xffffffff,
Data2: 0xffff,
Data3: 0xffff,
Data4: [8]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
}
}
// HvsockGUIDLoopback is the Loopback VmId for accepting connections to the same partition as the connector.
func HvsockGUIDLoopback() guid.GUID { // e0e16197-dd56-4a10-9195-5ee7a155a838
return guid.GUID{
Data1: 0xe0e16197,
Data2: 0xdd56,
Data3: 0x4a10,
Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38},
}
}
// HvsockGUIDSiloHost is the address of a silo's host partition:
// - The silo host of a hosted silo is the utility VM.
// - The silo host of a silo on a physical host is the physical host.
func HvsockGUIDSiloHost() guid.GUID { // 36bd0c5c-7276-4223-88ba-7d03b654c568
return guid.GUID{
Data1: 0x36bd0c5c,
Data2: 0x7276,
Data3: 0x4223,
Data4: [8]byte{0x88, 0xba, 0x7d, 0x03, 0xb6, 0x54, 0xc5, 0x68},
}
}
// HvsockGUIDChildren is the wildcard VmId for accepting connections from the connector's child partitions.
func HvsockGUIDChildren() guid.GUID { // 90db8b89-0d35-4f79-8ce9-49ea0ac8b7cd
return guid.GUID{
Data1: 0x90db8b89,
Data2: 0xd35,
Data3: 0x4f79,
Data4: [8]uint8{0x8c, 0xe9, 0x49, 0xea, 0xa, 0xc8, 0xb7, 0xcd},
}
}
// HvsockGUIDParent is the wildcard VmId for accepting connections from the connector's parent partition.
// Listening on this VmId accepts connection from:
// - Inside silos: silo host partition.
// - Inside hosted silo: host of the VM.
// - Inside VM: VM host.
// - Physical host: Not supported.
func HvsockGUIDParent() guid.GUID { // a42e7cda-d03f-480c-9cc2-a4de20abb878
return guid.GUID{
Data1: 0xa42e7cda,
Data2: 0xd03f,
Data3: 0x480c,
Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78},
}
}
// hvsockVsockServiceTemplate is the Service GUID used for the VSOCK protocol.
func hvsockVsockServiceTemplate() guid.GUID { // 00000000-facb-11e6-bd58-64006a7986d3
return guid.GUID{
Data2: 0xfacb,
Data3: 0x11e6,
Data4: [8]uint8{0xbd, 0x58, 0x64, 0x00, 0x6a, 0x79, 0x86, 0xd3},
}
}
// An HvsockAddr is an address for a AF_HYPERV socket.
type HvsockAddr struct {
VMID guid.GUID
ServiceID guid.GUID
}
type rawHvsockAddr struct {
Family uint16
_ uint16
VMID guid.GUID
ServiceID guid.GUID
}
var _ socket.RawSockaddr = &rawHvsockAddr{}
// Network returns the address's network name, "hvsock".
func (*HvsockAddr) Network() string {
return "hvsock"
}
func (addr *HvsockAddr) String() string {
return fmt.Sprintf("%s:%s", &addr.VMID, &addr.ServiceID)
}
// VsockServiceID returns an hvsock service ID corresponding to the specified AF_VSOCK port.
func VsockServiceID(port uint32) guid.GUID {
g := hvsockVsockServiceTemplate() // make a copy
g.Data1 = port
return g
}
func (addr *HvsockAddr) raw() rawHvsockAddr {
return rawHvsockAddr{
Family: afHVSock,
VMID: addr.VMID,
ServiceID: addr.ServiceID,
}
}
func (addr *HvsockAddr) fromRaw(raw *rawHvsockAddr) {
addr.VMID = raw.VMID
addr.ServiceID = raw.ServiceID
}
// Sockaddr returns a pointer to and the size of this struct.
//
// Implements the [socket.RawSockaddr] interface, and allows use in
// [socket.Bind] and [socket.ConnectEx].
func (r *rawHvsockAddr) Sockaddr() (unsafe.Pointer, int32, error) {
return unsafe.Pointer(r), int32(unsafe.Sizeof(rawHvsockAddr{})), nil
}
// Sockaddr interface allows use with `sockets.Bind()` and `.ConnectEx()`.
func (r *rawHvsockAddr) FromBytes(b []byte) error {
n := int(unsafe.Sizeof(rawHvsockAddr{}))
if len(b) < n {
return fmt.Errorf("got %d, want %d: %w", len(b), n, socket.ErrBufferSize)
}
copy(unsafe.Slice((*byte)(unsafe.Pointer(r)), n), b[:n])
if r.Family != afHVSock {
return fmt.Errorf("got %d, want %d: %w", r.Family, afHVSock, socket.ErrAddrFamily)
}
return nil
}
// HvsockListener is a socket listener for the AF_HYPERV address family.
type HvsockListener struct {
sock *win32File
addr HvsockAddr
}
var _ net.Listener = &HvsockListener{}
// HvsockConn is a connected socket of the AF_HYPERV address family.
type HvsockConn struct {
sock *win32File
local, remote HvsockAddr
}
var _ net.Conn = &HvsockConn{}
func newHVSocket() (*win32File, error) {
fd, err := windows.Socket(afHVSock, windows.SOCK_STREAM, 1)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
f, err := makeWin32File(fd)
if err != nil {
windows.Close(fd)
return nil, err
}
f.socket = true
return f, nil
}
// ListenHvsock listens for connections on the specified hvsock address.
func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) {
l := &HvsockListener{addr: *addr}
var sock *win32File
sock, err = newHVSocket()
if err != nil {
return nil, l.opErr("listen", err)
}
defer func() {
if err != nil {
_ = sock.Close()
}
}()
sa := addr.raw()
err = socket.Bind(sock.handle, &sa)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("socket", err))
}
err = windows.Listen(sock.handle, 16)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("listen", err))
}
return &HvsockListener{sock: sock, addr: *addr}, nil
}
func (l *HvsockListener) opErr(op string, err error) error {
return &net.OpError{Op: op, Net: "hvsock", Addr: &l.addr, Err: err}
}
// Addr returns the listener's network address.
func (l *HvsockListener) Addr() net.Addr {
return &l.addr
}
// Accept waits for the next connection and returns it.
func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("accept", err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
c, err := l.sock.prepareIO()
if err != nil {
return nil, l.opErr("accept", err)
}
defer l.sock.wg.Done()
// AcceptEx, per documentation, requires an extra 16 bytes per address.
//
// https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
const addrlen = uint32(16 + unsafe.Sizeof(rawHvsockAddr{}))
var addrbuf [addrlen * 2]byte
var bytes uint32
err = windows.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /* rxdatalen */, addrlen, addrlen, &bytes, &c.o)
if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil {
return nil, l.opErr("accept", os.NewSyscallError("acceptex", err))
}
conn := &HvsockConn{
sock: sock,
}
// The local address returned in the AcceptEx buffer is the same as the Listener socket's
// address. However, the service GUID reported by GetSockName is different from the Listeners
// socket, and is sometimes the same as the local address of the socket that dialed the
// address, with the service GUID.Data1 incremented, but othertimes is different.
// todo: does the local address matter? is the listener's address or the actual address appropriate?
conn.local.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[0])))
conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen])))
// initialize the accepted socket and update its properties with those of the listening socket
if err = windows.Setsockopt(sock.handle,
windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT,
(*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil {
return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err))
}
sock = nil
return conn, nil
}
// Close closes the listener, causing any pending Accept calls to fail.
func (l *HvsockListener) Close() error {
return l.sock.Close()
}
// HvsockDialer configures and dials a Hyper-V Socket (ie, [HvsockConn]).
type HvsockDialer struct {
// Deadline is the time the Dial operation must connect before erroring.
Deadline time.Time
// Retries is the number of additional connects to try if the connection times out, is refused,
// or the host is unreachable
Retries uint
// RetryWait is the time to wait after a connection error to retry
RetryWait time.Duration
rt *time.Timer // redial wait timer
}
// Dial the Hyper-V socket at addr.
//
// See [HvsockDialer.Dial] for more information.
func Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
return (&HvsockDialer{}).Dial(ctx, addr)
}
// Dial attempts to connect to the Hyper-V socket at addr, and returns a connection if successful.
// Will attempt (HvsockDialer).Retries if dialing fails, waiting (HvsockDialer).RetryWait between
// retries.
//
// Dialing can be cancelled either by providing (HvsockDialer).Deadline, or cancelling ctx.
func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
op := "dial"
// create the conn early to use opErr()
conn = &HvsockConn{
remote: *addr,
}
if !d.Deadline.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
defer cancel()
}
// preemptive timeout/cancellation check
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
sock, err := newHVSocket()
if err != nil {
return nil, conn.opErr(op, err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
sa := addr.raw()
err = socket.Bind(sock.handle, &sa)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("bind", err))
}
c, err := sock.prepareIO()
if err != nil {
return nil, conn.opErr(op, err)
}
defer sock.wg.Done()
var bytes uint32
for i := uint(0); i <= d.Retries; i++ {
err = socket.ConnectEx(
sock.handle,
&sa,
nil, // sendBuf
0, // sendDataLen
&bytes,
(*windows.Overlapped)(unsafe.Pointer(&c.o)))
_, err = sock.asyncIO(c, nil, bytes, err)
if i < d.Retries && canRedial(err) {
if err = d.redialWait(ctx); err == nil {
continue
}
}
break
}
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("connectex", err))
}
// update the connection properties, so shutdown can be used
if err = windows.Setsockopt(
sock.handle,
windows.SOL_SOCKET,
windows.SO_UPDATE_CONNECT_CONTEXT,
nil, // optvalue
0, // optlen
); err != nil {
return nil, conn.opErr(op, os.NewSyscallError("setsockopt", err))
}
// get the local name
var sal rawHvsockAddr
err = socket.GetSockName(sock.handle, &sal)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("getsockname", err))
}
conn.local.fromRaw(&sal)
// one last check for timeout, since asyncIO doesn't check the context
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
conn.sock = sock
sock = nil
return conn, nil
}
// redialWait waits before attempting to redial, resetting the timer as appropriate.
func (d *HvsockDialer) redialWait(ctx context.Context) (err error) {
if d.RetryWait == 0 {
return nil
}
if d.rt == nil {
d.rt = time.NewTimer(d.RetryWait)
} else {
// should already be stopped and drained
d.rt.Reset(d.RetryWait)
}
select {
case <-ctx.Done():
case <-d.rt.C:
return nil
}
// stop and drain the timer
if !d.rt.Stop() {
<-d.rt.C
}
return ctx.Err()
}
// assumes error is a plain, unwrapped windows.Errno provided by direct syscall.
func canRedial(err error) bool {
//nolint:errorlint // guaranteed to be an Errno
switch err {
case windows.WSAECONNREFUSED, windows.WSAENETUNREACH, windows.WSAETIMEDOUT,
windows.ERROR_CONNECTION_REFUSED, windows.ERROR_CONNECTION_UNAVAIL:
return true
default:
return false
}
}
func (conn *HvsockConn) opErr(op string, err error) error {
// translate from "file closed" to "socket closed"
if errors.Is(err, ErrFileClosed) {
err = socket.ErrSocketClosed
}
return &net.OpError{Op: op, Net: "hvsock", Source: &conn.local, Addr: &conn.remote, Err: err}
}
func (conn *HvsockConn) Read(b []byte) (int, error) {
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("read", err)
}
defer conn.sock.wg.Done()
buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var flags, bytes uint32
err = windows.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err)
if err != nil {
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsarecv", eno)
}
return 0, conn.opErr("read", err)
} else if n == 0 {
err = io.EOF
}
return n, err
}
func (conn *HvsockConn) Write(b []byte) (int, error) {
t := 0
for len(b) != 0 {
n, err := conn.write(b)
if err != nil {
return t + n, err
}
t += n
b = b[n:]
}
return t, nil
}
func (conn *HvsockConn) write(b []byte) (int, error) {
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("write", err)
}
defer conn.sock.wg.Done()
buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var bytes uint32
err = windows.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err)
if err != nil {
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsasend", eno)
}
return 0, conn.opErr("write", err)
}
return n, err
}
// Close closes the socket connection, failing any pending read or write calls.
func (conn *HvsockConn) Close() error {
return conn.sock.Close()
}
func (conn *HvsockConn) IsClosed() bool {
return conn.sock.IsClosed()
}
// shutdown disables sending or receiving on a socket.
func (conn *HvsockConn) shutdown(how int) error {
if conn.IsClosed() {
return socket.ErrSocketClosed
}
err := windows.Shutdown(conn.sock.handle, how)
if err != nil {
// If the connection was closed, shutdowns fail with "not connected"
if errors.Is(err, windows.WSAENOTCONN) ||
errors.Is(err, windows.WSAESHUTDOWN) {
err = socket.ErrSocketClosed
}
return os.NewSyscallError("shutdown", err)
}
return nil
}
// CloseRead shuts down the read end of the socket, preventing future read operations.
func (conn *HvsockConn) CloseRead() error {
err := conn.shutdown(windows.SHUT_RD)
if err != nil {
return conn.opErr("closeread", err)
}
return nil
}
// CloseWrite shuts down the write end of the socket, preventing future write operations and
// notifying the other endpoint that no more data will be written.
func (conn *HvsockConn) CloseWrite() error {
err := conn.shutdown(windows.SHUT_WR)
if err != nil {
return conn.opErr("closewrite", err)
}
return nil
}
// LocalAddr returns the local address of the connection.
func (conn *HvsockConn) LocalAddr() net.Addr {
return &conn.local
}
// RemoteAddr returns the remote address of the connection.
func (conn *HvsockConn) RemoteAddr() net.Addr {
return &conn.remote
}
// SetDeadline implements the net.Conn SetDeadline method.
func (conn *HvsockConn) SetDeadline(t time.Time) error {
// todo: implement `SetDeadline` for `win32File`
if err := conn.SetReadDeadline(t); err != nil {
return fmt.Errorf("set read deadline: %w", err)
}
if err := conn.SetWriteDeadline(t); err != nil {
return fmt.Errorf("set write deadline: %w", err)
}
return nil
}
// SetReadDeadline implements the net.Conn SetReadDeadline method.
func (conn *HvsockConn) SetReadDeadline(t time.Time) error {
return conn.sock.SetReadDeadline(t)
}
// SetWriteDeadline implements the net.Conn SetWriteDeadline method.
func (conn *HvsockConn) SetWriteDeadline(t time.Time) error {
return conn.sock.SetWriteDeadline(t)
}

View File

@@ -0,0 +1,2 @@
// This package contains Win32 filesystem functionality.
package fs

View File

@@ -0,0 +1,262 @@
//go:build windows
package fs
import (
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/stringbuffer"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go fs.go
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW
const NullHandle windows.Handle = 0
// AccessMask defines standard, specific, and generic rights.
//
// Used with CreateFile and NtCreateFile (and co.).
//
// Bitmask:
// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
// +---------------+---------------+-------------------------------+
// |G|G|G|G|Resvd|A| StandardRights| SpecificRights |
// |R|W|E|A| |S| | |
// +-+-------------+---------------+-------------------------------+
//
// GR Generic Read
// GW Generic Write
// GE Generic Exectue
// GA Generic All
// Resvd Reserved
// AS Access Security System
//
// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask
//
// https://learn.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
//
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants
type AccessMask = windows.ACCESS_MASK
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// Not actually any.
//
// For CreateFile: "query certain metadata such as file, directory, or device attributes without accessing that file or device"
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew#parameters
FILE_ANY_ACCESS AccessMask = 0
GENERIC_READ AccessMask = 0x8000_0000
GENERIC_WRITE AccessMask = 0x4000_0000
GENERIC_EXECUTE AccessMask = 0x2000_0000
GENERIC_ALL AccessMask = 0x1000_0000
ACCESS_SYSTEM_SECURITY AccessMask = 0x0100_0000
// Specific Object Access
// from ntioapi.h
FILE_READ_DATA AccessMask = (0x0001) // file & pipe
FILE_LIST_DIRECTORY AccessMask = (0x0001) // directory
FILE_WRITE_DATA AccessMask = (0x0002) // file & pipe
FILE_ADD_FILE AccessMask = (0x0002) // directory
FILE_APPEND_DATA AccessMask = (0x0004) // file
FILE_ADD_SUBDIRECTORY AccessMask = (0x0004) // directory
FILE_CREATE_PIPE_INSTANCE AccessMask = (0x0004) // named pipe
FILE_READ_EA AccessMask = (0x0008) // file & directory
FILE_READ_PROPERTIES AccessMask = FILE_READ_EA
FILE_WRITE_EA AccessMask = (0x0010) // file & directory
FILE_WRITE_PROPERTIES AccessMask = FILE_WRITE_EA
FILE_EXECUTE AccessMask = (0x0020) // file
FILE_TRAVERSE AccessMask = (0x0020) // directory
FILE_DELETE_CHILD AccessMask = (0x0040) // directory
FILE_READ_ATTRIBUTES AccessMask = (0x0080) // all
FILE_WRITE_ATTRIBUTES AccessMask = (0x0100) // all
FILE_ALL_ACCESS AccessMask = (STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0x1FF)
FILE_GENERIC_READ AccessMask = (STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE)
FILE_GENERIC_WRITE AccessMask = (STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE)
FILE_GENERIC_EXECUTE AccessMask = (STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE)
SPECIFIC_RIGHTS_ALL AccessMask = 0x0000FFFF
// Standard Access
// from ntseapi.h
DELETE AccessMask = 0x0001_0000
READ_CONTROL AccessMask = 0x0002_0000
WRITE_DAC AccessMask = 0x0004_0000
WRITE_OWNER AccessMask = 0x0008_0000
SYNCHRONIZE AccessMask = 0x0010_0000
STANDARD_RIGHTS_REQUIRED AccessMask = 0x000F_0000
STANDARD_RIGHTS_READ AccessMask = READ_CONTROL
STANDARD_RIGHTS_WRITE AccessMask = READ_CONTROL
STANDARD_RIGHTS_EXECUTE AccessMask = READ_CONTROL
STANDARD_RIGHTS_ALL AccessMask = 0x001F_0000
)
type FileShareMode uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
FILE_SHARE_NONE FileShareMode = 0x00
FILE_SHARE_READ FileShareMode = 0x01
FILE_SHARE_WRITE FileShareMode = 0x02
FILE_SHARE_DELETE FileShareMode = 0x04
FILE_SHARE_VALID_FLAGS FileShareMode = 0x07
)
type FileCreationDisposition uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winbase.h
CREATE_NEW FileCreationDisposition = 0x01
CREATE_ALWAYS FileCreationDisposition = 0x02
OPEN_EXISTING FileCreationDisposition = 0x03
OPEN_ALWAYS FileCreationDisposition = 0x04
TRUNCATE_EXISTING FileCreationDisposition = 0x05
)
// Create disposition values for NtCreate*
type NTFileCreationDisposition uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// From ntioapi.h
FILE_SUPERSEDE NTFileCreationDisposition = 0x00
FILE_OPEN NTFileCreationDisposition = 0x01
FILE_CREATE NTFileCreationDisposition = 0x02
FILE_OPEN_IF NTFileCreationDisposition = 0x03
FILE_OVERWRITE NTFileCreationDisposition = 0x04
FILE_OVERWRITE_IF NTFileCreationDisposition = 0x05
FILE_MAXIMUM_DISPOSITION NTFileCreationDisposition = 0x05
)
// CreateFile and co. take flags or attributes together as one parameter.
// Define alias until we can use generics to allow both
//
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants
type FileFlagOrAttribute uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winnt.h
FILE_FLAG_WRITE_THROUGH FileFlagOrAttribute = 0x8000_0000
FILE_FLAG_OVERLAPPED FileFlagOrAttribute = 0x4000_0000
FILE_FLAG_NO_BUFFERING FileFlagOrAttribute = 0x2000_0000
FILE_FLAG_RANDOM_ACCESS FileFlagOrAttribute = 0x1000_0000
FILE_FLAG_SEQUENTIAL_SCAN FileFlagOrAttribute = 0x0800_0000
FILE_FLAG_DELETE_ON_CLOSE FileFlagOrAttribute = 0x0400_0000
FILE_FLAG_BACKUP_SEMANTICS FileFlagOrAttribute = 0x0200_0000
FILE_FLAG_POSIX_SEMANTICS FileFlagOrAttribute = 0x0100_0000
FILE_FLAG_OPEN_REPARSE_POINT FileFlagOrAttribute = 0x0020_0000
FILE_FLAG_OPEN_NO_RECALL FileFlagOrAttribute = 0x0010_0000
FILE_FLAG_FIRST_PIPE_INSTANCE FileFlagOrAttribute = 0x0008_0000
)
// NtCreate* functions take a dedicated CreateOptions parameter.
//
// https://learn.microsoft.com/en-us/windows/win32/api/Winternl/nf-winternl-ntcreatefile
//
// https://learn.microsoft.com/en-us/windows/win32/devnotes/nt-create-named-pipe-file
type NTCreateOptions uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// From ntioapi.h
FILE_DIRECTORY_FILE NTCreateOptions = 0x0000_0001
FILE_WRITE_THROUGH NTCreateOptions = 0x0000_0002
FILE_SEQUENTIAL_ONLY NTCreateOptions = 0x0000_0004
FILE_NO_INTERMEDIATE_BUFFERING NTCreateOptions = 0x0000_0008
FILE_SYNCHRONOUS_IO_ALERT NTCreateOptions = 0x0000_0010
FILE_SYNCHRONOUS_IO_NONALERT NTCreateOptions = 0x0000_0020
FILE_NON_DIRECTORY_FILE NTCreateOptions = 0x0000_0040
FILE_CREATE_TREE_CONNECTION NTCreateOptions = 0x0000_0080
FILE_COMPLETE_IF_OPLOCKED NTCreateOptions = 0x0000_0100
FILE_NO_EA_KNOWLEDGE NTCreateOptions = 0x0000_0200
FILE_DISABLE_TUNNELING NTCreateOptions = 0x0000_0400
FILE_RANDOM_ACCESS NTCreateOptions = 0x0000_0800
FILE_DELETE_ON_CLOSE NTCreateOptions = 0x0000_1000
FILE_OPEN_BY_FILE_ID NTCreateOptions = 0x0000_2000
FILE_OPEN_FOR_BACKUP_INTENT NTCreateOptions = 0x0000_4000
FILE_NO_COMPRESSION NTCreateOptions = 0x0000_8000
)
type FileSQSFlag = FileFlagOrAttribute
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winbase.h
SECURITY_ANONYMOUS FileSQSFlag = FileSQSFlag(SecurityAnonymous << 16)
SECURITY_IDENTIFICATION FileSQSFlag = FileSQSFlag(SecurityIdentification << 16)
SECURITY_IMPERSONATION FileSQSFlag = FileSQSFlag(SecurityImpersonation << 16)
SECURITY_DELEGATION FileSQSFlag = FileSQSFlag(SecurityDelegation << 16)
SECURITY_SQOS_PRESENT FileSQSFlag = 0x0010_0000
SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F_0000
)
// GetFinalPathNameByHandle flags
//
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew#parameters
type GetFinalPathFlag uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
GetFinalPathDefaultFlag GetFinalPathFlag = 0x0
FILE_NAME_NORMALIZED GetFinalPathFlag = 0x0
FILE_NAME_OPENED GetFinalPathFlag = 0x8
VOLUME_NAME_DOS GetFinalPathFlag = 0x0
VOLUME_NAME_GUID GetFinalPathFlag = 0x1
VOLUME_NAME_NT GetFinalPathFlag = 0x2
VOLUME_NAME_NONE GetFinalPathFlag = 0x4
)
// getFinalPathNameByHandle facilitates calling the Windows API GetFinalPathNameByHandle
// with the given handle and flags. It transparently takes care of creating a buffer of the
// correct size for the call.
//
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew
func GetFinalPathNameByHandle(h windows.Handle, flags GetFinalPathFlag) (string, error) {
b := stringbuffer.NewWString()
//TODO: can loop infinitely if Win32 keeps returning the same (or a larger) n?
for {
n, err := windows.GetFinalPathNameByHandle(h, b.Pointer(), b.Cap(), uint32(flags))
if err != nil {
return "", err
}
// If the buffer wasn't large enough, n will be the total size needed (including null terminator).
// Resize and try again.
if n > b.Cap() {
b.ResizeTo(n)
continue
}
// If the buffer is large enough, n will be the size not including the null terminator.
// Convert to a Go string and return.
return b.String(), nil
}
}

View File

@@ -0,0 +1,12 @@
package fs
// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level
type SecurityImpersonationLevel int32 // C default enums underlying type is `int`, which is Go `int32`
// Impersonation levels
const (
SecurityAnonymous SecurityImpersonationLevel = 0
SecurityIdentification SecurityImpersonationLevel = 1
SecurityImpersonation SecurityImpersonationLevel = 2
SecurityDelegation SecurityImpersonationLevel = 3
)

View File

@@ -0,0 +1,61 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package fs
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procCreateFileW = modkernel32.NewProc("CreateFileW")
)
func CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(name)
if err != nil {
return
}
return _CreateFile(_p0, access, mode, sa, createmode, attrs, templatefile)
}
func _CreateFile(name *uint16, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) {
r0, _, e1 := syscall.SyscallN(procCreateFileW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile))
handle = windows.Handle(r0)
if handle == windows.InvalidHandle {
err = errnoErr(e1)
}
return
}

View File

@@ -0,0 +1,20 @@
package socket
import (
"unsafe"
)
// RawSockaddr allows structs to be used with [Bind] and [ConnectEx]. The
// struct must meet the Win32 sockaddr requirements specified here:
// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
//
// Specifically, the struct size must be least larger than an int16 (unsigned short)
// for the address family.
type RawSockaddr interface {
// Sockaddr returns a pointer to the RawSockaddr and its struct size, allowing
// for the RawSockaddr's data to be overwritten by syscalls (if necessary).
//
// It is the callers responsibility to validate that the values are valid; invalid
// pointers or size can cause a panic.
Sockaddr() (unsafe.Pointer, int32, error)
}

View File

@@ -0,0 +1,177 @@
//go:build windows
package socket
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"unsafe"
"github.com/Microsoft/go-winio/pkg/guid"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go socket.go
//sys getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getsockname
//sys getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getpeername
//sys bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const socketError = uintptr(^uint32(0))
var (
// todo(helsaawy): create custom error types to store the desired vs actual size and addr family?
ErrBufferSize = errors.New("buffer size")
ErrAddrFamily = errors.New("address family")
ErrInvalidPointer = errors.New("invalid pointer")
ErrSocketClosed = fmt.Errorf("socket closed: %w", net.ErrClosed)
)
// todo(helsaawy): replace these with generics, ie: GetSockName[S RawSockaddr](s windows.Handle) (S, error)
// GetSockName writes the local address of socket s to the [RawSockaddr] rsa.
// If rsa is not large enough, the [windows.WSAEFAULT] is returned.
func GetSockName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
// although getsockname returns WSAEFAULT if the buffer is too small, it does not set
// &l to the correct size, so--apart from doubling the buffer repeatedly--there is no remedy
return getsockname(s, ptr, &l)
}
// GetPeerName returns the remote address the socket is connected to.
//
// See [GetSockName] for more information.
func GetPeerName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return getpeername(s, ptr, &l)
}
func Bind(s windows.Handle, rsa RawSockaddr) (err error) {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return bind(s, ptr, l)
}
// "golang.org/x/sys/windows".ConnectEx and .Bind only accept internal implementations of the
// their sockaddr interface, so they cannot be used with HvsockAddr
// Replicate functionality here from
// https://cs.opensource.google/go/x/sys/+/master:windows/syscall_windows.go
// The function pointers to `AcceptEx`, `ConnectEx` and `GetAcceptExSockaddrs` must be loaded at
// runtime via a WSAIoctl call:
// https://docs.microsoft.com/en-us/windows/win32/api/Mswsock/nc-mswsock-lpfn_connectex#remarks
type runtimeFunc struct {
id guid.GUID
once sync.Once
addr uintptr
err error
}
func (f *runtimeFunc) Load() error {
f.once.Do(func() {
var s windows.Handle
s, f.err = windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP)
if f.err != nil {
return
}
defer windows.CloseHandle(s) //nolint:errcheck
var n uint32
f.err = windows.WSAIoctl(s,
windows.SIO_GET_EXTENSION_FUNCTION_POINTER,
(*byte)(unsafe.Pointer(&f.id)),
uint32(unsafe.Sizeof(f.id)),
(*byte)(unsafe.Pointer(&f.addr)),
uint32(unsafe.Sizeof(f.addr)),
&n,
nil, // overlapped
0, // completionRoutine
)
})
return f.err
}
var (
// todo: add `AcceptEx` and `GetAcceptExSockaddrs`
WSAID_CONNECTEX = guid.GUID{ //revive:disable-line:var-naming ALL_CAPS
Data1: 0x25a207b9,
Data2: 0xddf3,
Data3: 0x4660,
Data4: [8]byte{0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e},
}
connectExFunc = runtimeFunc{id: WSAID_CONNECTEX}
)
func ConnectEx(
fd windows.Handle,
rsa RawSockaddr,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) error {
if err := connectExFunc.Load(); err != nil {
return fmt.Errorf("failed to load ConnectEx function pointer: %w", err)
}
ptr, n, err := rsa.Sockaddr()
if err != nil {
return err
}
return connectEx(fd, ptr, n, sendBuf, sendDataLen, bytesSent, overlapped)
}
// BOOL LpfnConnectex(
// [in] SOCKET s,
// [in] const sockaddr *name,
// [in] int namelen,
// [in, optional] PVOID lpSendBuffer,
// [in] DWORD dwSendDataLength,
// [out] LPDWORD lpdwBytesSent,
// [in] LPOVERLAPPED lpOverlapped
// )
func connectEx(
s windows.Handle,
name unsafe.Pointer,
namelen int32,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) (err error) {
r1, _, e1 := syscall.SyscallN(connectExFunc.addr,
uintptr(s),
uintptr(name),
uintptr(namelen),
uintptr(unsafe.Pointer(sendBuf)),
uintptr(sendDataLen),
uintptr(unsafe.Pointer(bytesSent)),
uintptr(unsafe.Pointer(overlapped)),
)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return err
}

View File

@@ -0,0 +1,69 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package socket
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procbind = modws2_32.NewProc("bind")
procgetpeername = modws2_32.NewProc("getpeername")
procgetsockname = modws2_32.NewProc("getsockname")
)
func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.SyscallN(procbind.Addr(), uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.SyscallN(procgetpeername.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.SyscallN(procgetsockname.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

View File

@@ -0,0 +1,132 @@
package stringbuffer
import (
"sync"
"unicode/utf16"
)
// TODO: worth exporting and using in mkwinsyscall?
// Uint16BufferSize is the buffer size in the pool, chosen somewhat arbitrarily to accommodate
// large path strings:
// MAX_PATH (260) + size of volume GUID prefix (49) + null terminator = 310.
const MinWStringCap = 310
// use *[]uint16 since []uint16 creates an extra allocation where the slice header
// is copied to heap and then referenced via pointer in the interface header that sync.Pool
// stores.
var pathPool = sync.Pool{ // if go1.18+ adds Pool[T], use that to store []uint16 directly
New: func() interface{} {
b := make([]uint16, MinWStringCap)
return &b
},
}
func newBuffer() []uint16 { return *(pathPool.Get().(*[]uint16)) }
// freeBuffer copies the slice header data, and puts a pointer to that in the pool.
// This avoids taking a pointer to the slice header in WString, which can be set to nil.
func freeBuffer(b []uint16) { pathPool.Put(&b) }
// WString is a wide string buffer ([]uint16) meant for storing UTF-16 encoded strings
// for interacting with Win32 APIs.
// Sizes are specified as uint32 and not int.
//
// It is not thread safe.
type WString struct {
// type-def allows casting to []uint16 directly, use struct to prevent that and allow adding fields in the future.
// raw buffer
b []uint16
}
// NewWString returns a [WString] allocated from a shared pool with an
// initial capacity of at least [MinWStringCap].
// Since the buffer may have been previously used, its contents are not guaranteed to be empty.
//
// The buffer should be freed via [WString.Free]
func NewWString() *WString {
return &WString{
b: newBuffer(),
}
}
func (b *WString) Free() {
if b.empty() {
return
}
freeBuffer(b.b)
b.b = nil
}
// ResizeTo grows the buffer to at least c and returns the new capacity, freeing the
// previous buffer back into pool.
func (b *WString) ResizeTo(c uint32) uint32 {
// already sufficient (or n is 0)
if c <= b.Cap() {
return b.Cap()
}
if c <= MinWStringCap {
c = MinWStringCap
}
// allocate at-least double buffer size, as is done in [bytes.Buffer] and other places
if c <= 2*b.Cap() {
c = 2 * b.Cap()
}
b2 := make([]uint16, c)
if !b.empty() {
copy(b2, b.b)
freeBuffer(b.b)
}
b.b = b2
return c
}
// Buffer returns the underlying []uint16 buffer.
func (b *WString) Buffer() []uint16 {
if b.empty() {
return nil
}
return b.b
}
// Pointer returns a pointer to the first uint16 in the buffer.
// If the [WString.Free] has already been called, the pointer will be nil.
func (b *WString) Pointer() *uint16 {
if b.empty() {
return nil
}
return &b.b[0]
}
// String returns the returns the UTF-8 encoding of the UTF-16 string in the buffer.
//
// It assumes that the data is null-terminated.
func (b *WString) String() string {
// Using [windows.UTF16ToString] would require importing "golang.org/x/sys/windows"
// and would make this code Windows-only, which makes no sense.
// So copy UTF16ToString code into here.
// If other windows-specific code is added, switch to [windows.UTF16ToString]
s := b.b
for i, v := range s {
if v == 0 {
s = s[:i]
break
}
}
return string(utf16.Decode(s))
}
// Cap returns the underlying buffer capacity.
func (b *WString) Cap() uint32 {
if b.empty() {
return 0
}
return b.cap()
}
func (b *WString) cap() uint32 { return uint32(cap(b.b)) }
func (b *WString) empty() bool { return b == nil || b.cap() == 0 }

586
tools/tsgo/vendor/github.com/Microsoft/go-winio/pipe.go generated vendored Normal file
View File

@@ -0,0 +1,586 @@
//go:build windows
// +build windows
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"runtime"
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/fs"
)
//sys connectNamedPipe(pipe windows.Handle, o *windows.Overlapped) (err error) = ConnectNamedPipe
//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateNamedPipeW
//sys disconnectNamedPipe(pipe windows.Handle) (err error) = DisconnectNamedPipe
//sys getNamedPipeInfo(pipe windows.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe windows.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys ntCreateNamedPipeFile(pipe *windows.Handle, access ntAccessMask, oa *objectAttributes, iosb *ioStatusBlock, share ntFileShareMode, disposition ntFileCreationDisposition, options ntFileOptions, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl
type PipeConn interface {
net.Conn
Disconnect() error
Flush() error
}
// type aliases for mkwinsyscall code
type (
ntAccessMask = fs.AccessMask
ntFileShareMode = fs.FileShareMode
ntFileCreationDisposition = fs.NTFileCreationDisposition
ntFileOptions = fs.NTCreateOptions
)
type ioStatusBlock struct {
Status, Information uintptr
}
// typedef struct _OBJECT_ATTRIBUTES {
// ULONG Length;
// HANDLE RootDirectory;
// PUNICODE_STRING ObjectName;
// ULONG Attributes;
// PVOID SecurityDescriptor;
// PVOID SecurityQualityOfService;
// } OBJECT_ATTRIBUTES;
//
// https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-_object_attributes
type objectAttributes struct {
Length uintptr
RootDirectory uintptr
ObjectName *unicodeString
Attributes uintptr
SecurityDescriptor *securityDescriptor
SecurityQoS uintptr
}
type unicodeString struct {
Length uint16
MaximumLength uint16
Buffer uintptr
}
// typedef struct _SECURITY_DESCRIPTOR {
// BYTE Revision;
// BYTE Sbz1;
// SECURITY_DESCRIPTOR_CONTROL Control;
// PSID Owner;
// PSID Group;
// PACL Sacl;
// PACL Dacl;
// } SECURITY_DESCRIPTOR, *PISECURITY_DESCRIPTOR;
//
// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-security_descriptor
type securityDescriptor struct {
Revision byte
Sbz1 byte
Control uint16
Owner uintptr
Group uintptr
Sacl uintptr //revive:disable-line:var-naming SACL, not Sacl
Dacl uintptr //revive:disable-line:var-naming DACL, not Dacl
}
type ntStatus int32
func (status ntStatus) Err() error {
if status >= 0 {
return nil
}
return rtlNtStatusToDosError(status)
}
var (
// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
ErrPipeListenerClosed = net.ErrClosed
errPipeWriteClosed = errors.New("pipe has been closed for write")
)
type win32Pipe struct {
*win32File
path string
}
var _ PipeConn = (*win32Pipe)(nil)
type win32MessageBytePipe struct {
win32Pipe
writeClosed bool
readEOF bool
}
type pipeAddress string
func (f *win32Pipe) LocalAddr() net.Addr {
return pipeAddress(f.path)
}
func (f *win32Pipe) RemoteAddr() net.Addr {
return pipeAddress(f.path)
}
func (f *win32Pipe) SetDeadline(t time.Time) error {
if err := f.SetReadDeadline(t); err != nil {
return err
}
return f.SetWriteDeadline(t)
}
func (f *win32Pipe) Disconnect() error {
return disconnectNamedPipe(f.win32File.handle)
}
// CloseWrite closes the write side of a message pipe in byte mode.
func (f *win32MessageBytePipe) CloseWrite() error {
if f.writeClosed {
return errPipeWriteClosed
}
err := f.win32File.Flush()
if err != nil {
return err
}
_, err = f.win32File.Write(nil)
if err != nil {
return err
}
f.writeClosed = true
return nil
}
// Write writes bytes to a message pipe in byte mode. Zero-byte writes are ignored, since
// they are used to implement CloseWrite().
func (f *win32MessageBytePipe) Write(b []byte) (int, error) {
if f.writeClosed {
return 0, errPipeWriteClosed
}
if len(b) == 0 {
return 0, nil
}
return f.win32File.Write(b)
}
// Read reads bytes from a message pipe in byte mode. A read of a zero-byte message on a message
// mode pipe will return io.EOF, as will all subsequent reads.
func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
if f.readEOF {
return 0, io.EOF
}
n, err := f.win32File.Read(b)
if err == io.EOF { //nolint:errorlint
// If this was the result of a zero-byte read, then
// it is possible that the read was due to a zero-size
// message. Since we are simulating CloseWrite with a
// zero-byte message, ensure that all future Read() calls
// also return EOF.
f.readEOF = true
} else if err == windows.ERROR_MORE_DATA { //nolint:errorlint // err is Errno
// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
// and the message still has more bytes. Treat this as a success, since
// this package presents all named pipes as byte streams.
err = nil
}
return n, err
}
func (pipeAddress) Network() string {
return "pipe"
}
func (s pipeAddress) String() string {
return string(s)
}
// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
func tryDialPipe(ctx context.Context, path *string, access fs.AccessMask, impLevel PipeImpLevel) (windows.Handle, error) {
for {
select {
case <-ctx.Done():
return windows.Handle(0), ctx.Err()
default:
h, err := fs.CreateFile(*path,
access,
0, // mode
nil, // security attributes
fs.OPEN_EXISTING,
fs.FILE_FLAG_OVERLAPPED|fs.SECURITY_SQOS_PRESENT|fs.FileSQSFlag(impLevel),
0, // template file handle
)
if err == nil {
return h, nil
}
if err != windows.ERROR_PIPE_BUSY { //nolint:errorlint // err is Errno
return h, &os.PathError{Err: err, Op: "open", Path: *path}
}
// Wait 10 msec and try again. This is a rather simplistic
// view, as we always try each 10 milliseconds.
time.Sleep(10 * time.Millisecond)
}
}
}
// DialPipe connects to a named pipe by path, timing out if the connection
// takes longer than the specified duration. If timeout is nil, then we use
// a default timeout of 2 seconds. (We do not use WaitNamedPipe.)
func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
var absTimeout time.Time
if timeout != nil {
absTimeout = time.Now().Add(*timeout)
} else {
absTimeout = time.Now().Add(2 * time.Second)
}
ctx, cancel := context.WithDeadline(context.Background(), absTimeout)
defer cancel()
conn, err := DialPipeContext(ctx, path)
if errors.Is(err, context.DeadlineExceeded) {
return nil, ErrTimeout
}
return conn, err
}
// DialPipeContext attempts to connect to a named pipe by `path` until `ctx`
// cancellation or timeout.
func DialPipeContext(ctx context.Context, path string) (net.Conn, error) {
return DialPipeAccess(ctx, path, uint32(fs.GENERIC_READ|fs.GENERIC_WRITE))
}
// PipeImpLevel is an enumeration of impersonation levels that may be set
// when calling DialPipeAccessImpersonation.
type PipeImpLevel uint32
const (
PipeImpLevelAnonymous = PipeImpLevel(fs.SECURITY_ANONYMOUS)
PipeImpLevelIdentification = PipeImpLevel(fs.SECURITY_IDENTIFICATION)
PipeImpLevelImpersonation = PipeImpLevel(fs.SECURITY_IMPERSONATION)
PipeImpLevelDelegation = PipeImpLevel(fs.SECURITY_DELEGATION)
)
// DialPipeAccess attempts to connect to a named pipe by `path` with `access` until `ctx`
// cancellation or timeout.
func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, error) {
return DialPipeAccessImpLevel(ctx, path, access, PipeImpLevelAnonymous)
}
// DialPipeAccessImpLevel attempts to connect to a named pipe by `path` with
// `access` at `impLevel` until `ctx` cancellation or timeout. The other
// DialPipe* implementations use PipeImpLevelAnonymous.
func DialPipeAccessImpLevel(ctx context.Context, path string, access uint32, impLevel PipeImpLevel) (net.Conn, error) {
var err error
var h windows.Handle
h, err = tryDialPipe(ctx, &path, fs.AccessMask(access), impLevel)
if err != nil {
return nil, err
}
var flags uint32
err = getNamedPipeInfo(h, &flags, nil, nil, nil)
if err != nil {
return nil, err
}
f, err := makeWin32File(h)
if err != nil {
windows.Close(h)
return nil, err
}
// If the pipe is in message mode, return a message byte pipe, which
// supports CloseWrite().
if flags&windows.PIPE_TYPE_MESSAGE != 0 {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: f, path: path},
}, nil
}
return &win32Pipe{win32File: f, path: path}, nil
}
type acceptResponse struct {
f *win32File
err error
}
type win32PipeListener struct {
firstHandle windows.Handle
path string
config PipeConfig
acceptCh chan (chan acceptResponse)
closeCh chan int
doneCh chan int
}
func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (windows.Handle, error) {
path16, err := windows.UTF16FromString(path)
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
var oa objectAttributes
oa.Length = unsafe.Sizeof(oa)
var ntPath unicodeString
if err := rtlDosPathNameToNtPathName(&path16[0],
&ntPath,
0,
0,
).Err(); err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
defer windows.LocalFree(windows.Handle(ntPath.Buffer)) //nolint:errcheck
oa.ObjectName = &ntPath
oa.Attributes = windows.OBJ_CASE_INSENSITIVE
// The security descriptor is only needed for the first pipe.
if first {
if sd != nil {
//todo: does `sdb` need to be allocated on the heap, or can go allocate it?
l := uint32(len(sd))
sdb, err := windows.LocalAlloc(0, l)
if err != nil {
return 0, fmt.Errorf("LocalAlloc for security descriptor with of length %d: %w", l, err)
}
defer windows.LocalFree(windows.Handle(sdb)) //nolint:errcheck
copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
} else {
// Construct the default named pipe security descriptor.
var dacl uintptr
if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
return 0, fmt.Errorf("getting default named pipe ACL: %w", err)
}
defer windows.LocalFree(windows.Handle(dacl)) //nolint:errcheck
sdb := &securityDescriptor{
Revision: 1,
Control: windows.SE_DACL_PRESENT,
Dacl: dacl,
}
oa.SecurityDescriptor = sdb
}
}
typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
if c.MessageMode {
typ |= windows.FILE_PIPE_MESSAGE_TYPE
}
disposition := fs.FILE_OPEN
access := fs.GENERIC_READ | fs.GENERIC_WRITE | fs.SYNCHRONIZE
if first {
disposition = fs.FILE_CREATE
// By not asking for read or write access, the named pipe file system
// will put this pipe into an initially disconnected state, blocking
// client connections until the next call with first == false.
access = fs.SYNCHRONIZE
}
timeout := int64(-50 * 10000) // 50ms
var (
h windows.Handle
iosb ioStatusBlock
)
err = ntCreateNamedPipeFile(&h,
access,
&oa,
&iosb,
fs.FILE_SHARE_READ|fs.FILE_SHARE_WRITE,
disposition,
0,
typ,
0,
0,
0xffffffff,
uint32(c.InputBufferSize),
uint32(c.OutputBufferSize),
&timeout).Err()
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
runtime.KeepAlive(ntPath)
return h, nil
}
func (l *win32PipeListener) makeServerPipe() (*win32File, error) {
h, err := makeServerPipeHandle(l.path, nil, &l.config, false)
if err != nil {
return nil, err
}
f, err := makeWin32File(h)
if err != nil {
windows.Close(h)
return nil, err
}
return f, nil
}
func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
p, err := l.makeServerPipe()
if err != nil {
return nil, err
}
// Wait for the client to connect.
ch := make(chan error)
go func(p *win32File) {
ch <- connectPipe(p)
}(p)
select {
case err = <-ch:
if err != nil {
p.Close()
p = nil
}
case <-l.closeCh:
// Abort the connect request by closing the handle.
p.Close()
p = nil
err = <-ch
if err == nil || err == ErrFileClosed { //nolint:errorlint // err is Errno
err = ErrPipeListenerClosed
}
}
return p, err
}
func (l *win32PipeListener) listenerRoutine() {
closed := false
for !closed {
select {
case <-l.closeCh:
closed = true
case responseCh := <-l.acceptCh:
var (
p *win32File
err error
)
for {
p, err = l.makeConnectedServerPipe()
// If the connection was immediately closed by the client, try
// again.
if err != windows.ERROR_NO_DATA { //nolint:errorlint // err is Errno
break
}
}
responseCh <- acceptResponse{p, err}
closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno
}
}
windows.Close(l.firstHandle)
l.firstHandle = 0
// Notify Close() and Accept() callers that the handle has been closed.
close(l.doneCh)
}
// PipeConfig contain configuration for the pipe listener.
type PipeConfig struct {
// SecurityDescriptor contains a Windows security descriptor in SDDL format.
SecurityDescriptor string
// MessageMode determines whether the pipe is in byte or message mode. In either
// case the pipe is read in byte mode by default. The only practical difference in
// this implementation is that CloseWrite() is only supported for message mode pipes;
// CloseWrite() is implemented as a zero-byte write, but zero-byte writes are only
// transferred to the reader (and returned as io.EOF in this implementation)
// when the pipe is in message mode.
MessageMode bool
// InputBufferSize specifies the size of the input buffer, in bytes.
InputBufferSize int32
// OutputBufferSize specifies the size of the output buffer, in bytes.
OutputBufferSize int32
}
// ListenPipe creates a listener on a Windows named pipe path, e.g. \\.\pipe\mypipe.
// The pipe must not already exist.
func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
var (
sd []byte
err error
)
if c == nil {
c = &PipeConfig{}
}
if c.SecurityDescriptor != "" {
sd, err = SddlToSecurityDescriptor(c.SecurityDescriptor)
if err != nil {
return nil, err
}
}
h, err := makeServerPipeHandle(path, sd, c, true)
if err != nil {
return nil, err
}
l := &win32PipeListener{
firstHandle: h,
path: path,
config: *c,
acceptCh: make(chan (chan acceptResponse)),
closeCh: make(chan int),
doneCh: make(chan int),
}
go l.listenerRoutine()
return l, nil
}
func connectPipe(p *win32File) error {
c, err := p.prepareIO()
if err != nil {
return err
}
defer p.wg.Done()
err = connectNamedPipe(p.handle, &c.o)
_, err = p.asyncIO(c, nil, 0, err)
if err != nil && err != windows.ERROR_PIPE_CONNECTED { //nolint:errorlint // err is Errno
return err
}
return nil
}
func (l *win32PipeListener) Accept() (net.Conn, error) {
ch := make(chan acceptResponse)
select {
case l.acceptCh <- ch:
response := <-ch
err := response.err
if err != nil {
return nil, err
}
if l.config.MessageMode {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: response.f, path: l.path},
}, nil
}
return &win32Pipe{win32File: response.f, path: l.path}, nil
case <-l.doneCh:
return nil, ErrPipeListenerClosed
}
}
func (l *win32PipeListener) Close() error {
select {
case l.closeCh <- 1:
<-l.doneCh
case <-l.doneCh:
}
return nil
}
func (l *win32PipeListener) Addr() net.Addr {
return pipeAddress(l.path)
}

View File

@@ -0,0 +1,232 @@
// Package guid provides a GUID type. The backing structure for a GUID is
// identical to that used by the golang.org/x/sys/windows GUID type.
// There are two main binary encodings used for a GUID, the big-endian encoding,
// and the Windows (mixed-endian) encoding. See here for details:
// https://en.wikipedia.org/wiki/Universally_unique_identifier#Encoding
package guid
import (
"crypto/rand"
"crypto/sha1" //nolint:gosec // not used for secure application
"encoding"
"encoding/binary"
"fmt"
"strconv"
)
//go:generate go run golang.org/x/tools/cmd/stringer -type=Variant -trimprefix=Variant -linecomment
// Variant specifies which GUID variant (or "type") of the GUID. It determines
// how the entirety of the rest of the GUID is interpreted.
type Variant uint8
// The variants specified by RFC 4122 section 4.1.1.
const (
// VariantUnknown specifies a GUID variant which does not conform to one of
// the variant encodings specified in RFC 4122.
VariantUnknown Variant = iota
VariantNCS
VariantRFC4122 // RFC 4122
VariantMicrosoft
VariantFuture
)
// Version specifies how the bits in the GUID were generated. For instance, a
// version 4 GUID is randomly generated, and a version 5 is generated from the
// hash of an input string.
type Version uint8
func (v Version) String() string {
return strconv.FormatUint(uint64(v), 10)
}
var _ = (encoding.TextMarshaler)(GUID{})
var _ = (encoding.TextUnmarshaler)(&GUID{})
// NewV4 returns a new version 4 (pseudorandom) GUID, as defined by RFC 4122.
func NewV4() (GUID, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return GUID{}, err
}
g := FromArray(b)
g.setVersion(4) // Version 4 means randomly generated.
g.setVariant(VariantRFC4122)
return g, nil
}
// NewV5 returns a new version 5 (generated from a string via SHA-1 hashing)
// GUID, as defined by RFC 4122. The RFC is unclear on the encoding of the name,
// and the sample code treats it as a series of bytes, so we do the same here.
//
// Some implementations, such as those found on Windows, treat the name as a
// big-endian UTF16 stream of bytes. If that is desired, the string can be
// encoded as such before being passed to this function.
func NewV5(namespace GUID, name []byte) (GUID, error) {
b := sha1.New() //nolint:gosec // not used for secure application
namespaceBytes := namespace.ToArray()
b.Write(namespaceBytes[:])
b.Write(name)
a := [16]byte{}
copy(a[:], b.Sum(nil))
g := FromArray(a)
g.setVersion(5) // Version 5 means generated from a string.
g.setVariant(VariantRFC4122)
return g, nil
}
func fromArray(b [16]byte, order binary.ByteOrder) GUID {
var g GUID
g.Data1 = order.Uint32(b[0:4])
g.Data2 = order.Uint16(b[4:6])
g.Data3 = order.Uint16(b[6:8])
copy(g.Data4[:], b[8:16])
return g
}
func (g GUID) toArray(order binary.ByteOrder) [16]byte {
b := [16]byte{}
order.PutUint32(b[0:4], g.Data1)
order.PutUint16(b[4:6], g.Data2)
order.PutUint16(b[6:8], g.Data3)
copy(b[8:16], g.Data4[:])
return b
}
// FromArray constructs a GUID from a big-endian encoding array of 16 bytes.
func FromArray(b [16]byte) GUID {
return fromArray(b, binary.BigEndian)
}
// ToArray returns an array of 16 bytes representing the GUID in big-endian
// encoding.
func (g GUID) ToArray() [16]byte {
return g.toArray(binary.BigEndian)
}
// FromWindowsArray constructs a GUID from a Windows encoding array of bytes.
func FromWindowsArray(b [16]byte) GUID {
return fromArray(b, binary.LittleEndian)
}
// ToWindowsArray returns an array of 16 bytes representing the GUID in Windows
// encoding.
func (g GUID) ToWindowsArray() [16]byte {
return g.toArray(binary.LittleEndian)
}
func (g GUID) String() string {
return fmt.Sprintf(
"%08x-%04x-%04x-%04x-%012x",
g.Data1,
g.Data2,
g.Data3,
g.Data4[:2],
g.Data4[2:])
}
// FromString parses a string containing a GUID and returns the GUID. The only
// format currently supported is the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`
// format.
func FromString(s string) (GUID, error) {
if len(s) != 36 {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
var g GUID
data1, err := strconv.ParseUint(s[0:8], 16, 32)
if err != nil {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
g.Data1 = uint32(data1)
data2, err := strconv.ParseUint(s[9:13], 16, 16)
if err != nil {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
g.Data2 = uint16(data2)
data3, err := strconv.ParseUint(s[14:18], 16, 16)
if err != nil {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
g.Data3 = uint16(data3)
for i, x := range []int{19, 21, 24, 26, 28, 30, 32, 34} {
v, err := strconv.ParseUint(s[x:x+2], 16, 8)
if err != nil {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
g.Data4[i] = uint8(v)
}
return g, nil
}
func (g *GUID) setVariant(v Variant) {
d := g.Data4[0]
switch v {
case VariantNCS:
d = (d & 0x7f)
case VariantRFC4122:
d = (d & 0x3f) | 0x80
case VariantMicrosoft:
d = (d & 0x1f) | 0xc0
case VariantFuture:
d = (d & 0x0f) | 0xe0
case VariantUnknown:
fallthrough
default:
panic(fmt.Sprintf("invalid variant: %d", v))
}
g.Data4[0] = d
}
// Variant returns the GUID variant, as defined in RFC 4122.
func (g GUID) Variant() Variant {
b := g.Data4[0]
if b&0x80 == 0 {
return VariantNCS
} else if b&0xc0 == 0x80 {
return VariantRFC4122
} else if b&0xe0 == 0xc0 {
return VariantMicrosoft
} else if b&0xe0 == 0xe0 {
return VariantFuture
}
return VariantUnknown
}
func (g *GUID) setVersion(v Version) {
g.Data3 = (g.Data3 & 0x0fff) | (uint16(v) << 12)
}
// Version returns the GUID version, as defined in RFC 4122.
func (g GUID) Version() Version {
return Version((g.Data3 & 0xF000) >> 12)
}
// MarshalText returns the textual representation of the GUID.
func (g GUID) MarshalText() ([]byte, error) {
return []byte(g.String()), nil
}
// UnmarshalText takes the textual representation of a GUID, and unmarhals it
// into this GUID.
func (g *GUID) UnmarshalText(text []byte) error {
g2, err := FromString(string(text))
if err != nil {
return err
}
*g = g2
return nil
}

View File

@@ -0,0 +1,16 @@
//go:build !windows
// +build !windows
package guid
// GUID represents a GUID/UUID. It has the same structure as
// golang.org/x/sys/windows.GUID so that it can be used with functions expecting
// that type. It is defined as its own type as that is only available to builds
// targeted at `windows`. The representation matches that used by native Windows
// code.
type GUID struct {
Data1 uint32
Data2 uint16
Data3 uint16
Data4 [8]byte
}

View File

@@ -0,0 +1,13 @@
//go:build windows
// +build windows
package guid
import "golang.org/x/sys/windows"
// GUID represents a GUID/UUID. It has the same structure as
// golang.org/x/sys/windows.GUID so that it can be used with functions expecting
// that type. It is defined as its own type so that stringification and
// marshaling can be supported. The representation matches that used by native
// Windows code.
type GUID windows.GUID

View File

@@ -0,0 +1,27 @@
// Code generated by "stringer -type=Variant -trimprefix=Variant -linecomment"; DO NOT EDIT.
package guid
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[VariantUnknown-0]
_ = x[VariantNCS-1]
_ = x[VariantRFC4122-2]
_ = x[VariantMicrosoft-3]
_ = x[VariantFuture-4]
}
const _Variant_name = "UnknownNCSRFC 4122MicrosoftFuture"
var _Variant_index = [...]uint8{0, 7, 10, 18, 27, 33}
func (i Variant) String() string {
if i >= Variant(len(_Variant_index)-1) {
return "Variant(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Variant_name[_Variant_index[i]:_Variant_index[i+1]]
}

View File

@@ -0,0 +1,196 @@
//go:build windows
// +build windows
package winio
import (
"bytes"
"encoding/binary"
"fmt"
"runtime"
"sync"
"unicode/utf16"
"golang.org/x/sys/windows"
)
//sys adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) [true] = advapi32.AdjustTokenPrivileges
//sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf
//sys revertToSelf() (err error) = advapi32.RevertToSelf
//sys openThreadToken(thread windows.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken
//sys getCurrentThread() (h windows.Handle) = GetCurrentThread
//sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW
//sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW
//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
const (
//revive:disable-next-line:var-naming ALL_CAPS
SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED
//revive:disable-next-line:var-naming ALL_CAPS
ERROR_NOT_ALL_ASSIGNED windows.Errno = windows.ERROR_NOT_ALL_ASSIGNED
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
var (
privNames = make(map[string]uint64)
privNameMutex sync.Mutex
)
// PrivilegeError represents an error enabling privileges.
type PrivilegeError struct {
privileges []uint64
}
func (e *PrivilegeError) Error() string {
s := "Could not enable privilege "
if len(e.privileges) > 1 {
s = "Could not enable privileges "
}
for i, p := range e.privileges {
if i != 0 {
s += ", "
}
s += `"`
s += getPrivilegeName(p)
s += `"`
}
return s
}
// RunWithPrivilege enables a single privilege for a function call.
func RunWithPrivilege(name string, fn func() error) error {
return RunWithPrivileges([]string{name}, fn)
}
// RunWithPrivileges enables privileges for a function call.
func RunWithPrivileges(names []string, fn func() error) error {
privileges, err := mapPrivileges(names)
if err != nil {
return err
}
runtime.LockOSThread()
defer runtime.UnlockOSThread()
token, err := newThreadToken()
if err != nil {
return err
}
defer releaseThreadToken(token)
err = adjustPrivileges(token, privileges, SE_PRIVILEGE_ENABLED)
if err != nil {
return err
}
return fn()
}
func mapPrivileges(names []string) ([]uint64, error) {
privileges := make([]uint64, 0, len(names))
privNameMutex.Lock()
defer privNameMutex.Unlock()
for _, name := range names {
p, ok := privNames[name]
if !ok {
err := lookupPrivilegeValue("", name, &p)
if err != nil {
return nil, err
}
privNames[name] = p
}
privileges = append(privileges, p)
}
return privileges, nil
}
// EnableProcessPrivileges enables privileges globally for the process.
func EnableProcessPrivileges(names []string) error {
return enableDisableProcessPrivilege(names, SE_PRIVILEGE_ENABLED)
}
// DisableProcessPrivileges disables privileges globally for the process.
func DisableProcessPrivileges(names []string) error {
return enableDisableProcessPrivilege(names, 0)
}
func enableDisableProcessPrivilege(names []string, action uint32) error {
privileges, err := mapPrivileges(names)
if err != nil {
return err
}
p := windows.CurrentProcess()
var token windows.Token
err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
if err != nil {
return err
}
defer token.Close()
return adjustPrivileges(token, privileges, action)
}
func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
var b bytes.Buffer
_ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
for _, p := range privileges {
_ = binary.Write(&b, binary.LittleEndian, p)
_ = binary.Write(&b, binary.LittleEndian, action)
}
prevState := make([]byte, b.Len())
reqSize := uint32(0)
success, err := adjustTokenPrivileges(token, false, &b.Bytes()[0], uint32(len(prevState)), &prevState[0], &reqSize)
if !success {
return err
}
if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno
return &PrivilegeError{privileges}
}
return nil
}
func getPrivilegeName(luid uint64) string {
var nameBuffer [256]uint16
bufSize := uint32(len(nameBuffer))
err := lookupPrivilegeName("", &luid, &nameBuffer[0], &bufSize)
if err != nil {
return fmt.Sprintf("<unknown privilege %d>", luid)
}
var displayNameBuffer [256]uint16
displayBufSize := uint32(len(displayNameBuffer))
var langID uint32
err = lookupPrivilegeDisplayName("", &nameBuffer[0], &displayNameBuffer[0], &displayBufSize, &langID)
if err != nil {
return fmt.Sprintf("<unknown privilege %s>", string(utf16.Decode(nameBuffer[:bufSize])))
}
return string(utf16.Decode(displayNameBuffer[:displayBufSize]))
}
func newThreadToken() (windows.Token, error) {
err := impersonateSelf(windows.SecurityImpersonation)
if err != nil {
return 0, err
}
var token windows.Token
err = openThreadToken(getCurrentThread(), windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, false, &token)
if err != nil {
rerr := revertToSelf()
if rerr != nil {
panic(rerr)
}
return 0, err
}
return token, nil
}
func releaseThreadToken(h windows.Token) {
err := revertToSelf()
if err != nil {
panic(err)
}
h.Close()
}

View File

@@ -0,0 +1,131 @@
//go:build windows
// +build windows
package winio
import (
"bytes"
"encoding/binary"
"fmt"
"strings"
"unicode/utf16"
"unsafe"
)
const (
reparseTagMountPoint = 0xA0000003
reparseTagSymlink = 0xA000000C
)
type reparseDataBuffer struct {
ReparseTag uint32
ReparseDataLength uint16
Reserved uint16
SubstituteNameOffset uint16
SubstituteNameLength uint16
PrintNameOffset uint16
PrintNameLength uint16
}
// ReparsePoint describes a Win32 symlink or mount point.
type ReparsePoint struct {
Target string
IsMountPoint bool
}
// UnsupportedReparsePointError is returned when trying to decode a non-symlink or
// mount point reparse point.
type UnsupportedReparsePointError struct {
Tag uint32
}
func (e *UnsupportedReparsePointError) Error() string {
return fmt.Sprintf("unsupported reparse point %x", e.Tag)
}
// DecodeReparsePoint decodes a Win32 REPARSE_DATA_BUFFER structure containing either a symlink
// or a mount point.
func DecodeReparsePoint(b []byte) (*ReparsePoint, error) {
tag := binary.LittleEndian.Uint32(b[0:4])
return DecodeReparsePointData(tag, b[8:])
}
func DecodeReparsePointData(tag uint32, b []byte) (*ReparsePoint, error) {
isMountPoint := false
switch tag {
case reparseTagMountPoint:
isMountPoint = true
case reparseTagSymlink:
default:
return nil, &UnsupportedReparsePointError{tag}
}
nameOffset := 8 + binary.LittleEndian.Uint16(b[4:6])
if !isMountPoint {
nameOffset += 4
}
nameLength := binary.LittleEndian.Uint16(b[6:8])
name := make([]uint16, nameLength/2)
err := binary.Read(bytes.NewReader(b[nameOffset:nameOffset+nameLength]), binary.LittleEndian, &name)
if err != nil {
return nil, err
}
return &ReparsePoint{string(utf16.Decode(name)), isMountPoint}, nil
}
func isDriveLetter(c byte) bool {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
// EncodeReparsePoint encodes a Win32 REPARSE_DATA_BUFFER structure describing a symlink or
// mount point.
func EncodeReparsePoint(rp *ReparsePoint) []byte {
// Generate an NT path and determine if this is a relative path.
var ntTarget string
relative := false
if strings.HasPrefix(rp.Target, `\\?\`) {
ntTarget = `\??\` + rp.Target[4:]
} else if strings.HasPrefix(rp.Target, `\\`) {
ntTarget = `\??\UNC\` + rp.Target[2:]
} else if len(rp.Target) >= 2 && isDriveLetter(rp.Target[0]) && rp.Target[1] == ':' {
ntTarget = `\??\` + rp.Target
} else {
ntTarget = rp.Target
relative = true
}
// The paths must be NUL-terminated even though they are counted strings.
target16 := utf16.Encode([]rune(rp.Target + "\x00"))
ntTarget16 := utf16.Encode([]rune(ntTarget + "\x00"))
size := int(unsafe.Sizeof(reparseDataBuffer{})) - 8
size += len(ntTarget16)*2 + len(target16)*2
tag := uint32(reparseTagMountPoint)
if !rp.IsMountPoint {
tag = reparseTagSymlink
size += 4 // Add room for symlink flags
}
data := reparseDataBuffer{
ReparseTag: tag,
ReparseDataLength: uint16(size),
SubstituteNameOffset: 0,
SubstituteNameLength: uint16((len(ntTarget16) - 1) * 2),
PrintNameOffset: uint16(len(ntTarget16) * 2),
PrintNameLength: uint16((len(target16) - 1) * 2),
}
var b bytes.Buffer
_ = binary.Write(&b, binary.LittleEndian, &data)
if !rp.IsMountPoint {
flags := uint32(0)
if relative {
flags |= 1
}
_ = binary.Write(&b, binary.LittleEndian, flags)
}
_ = binary.Write(&b, binary.LittleEndian, ntTarget16)
_ = binary.Write(&b, binary.LittleEndian, target16)
return b.Bytes()
}

133
tools/tsgo/vendor/github.com/Microsoft/go-winio/sd.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
//go:build windows
// +build windows
package winio
import (
"errors"
"fmt"
"unsafe"
"golang.org/x/sys/windows"
)
//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
//sys lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountSidW
//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
//sys convertStringSidToSid(str *uint16, sid **byte) (err error) = advapi32.ConvertStringSidToSidW
type AccountLookupError struct {
Name string
Err error
}
func (e *AccountLookupError) Error() string {
if e.Name == "" {
return "lookup account: empty account name specified"
}
var s string
switch {
case errors.Is(e.Err, windows.ERROR_INVALID_SID):
s = "the security ID structure is invalid"
case errors.Is(e.Err, windows.ERROR_NONE_MAPPED):
s = "not found"
default:
s = e.Err.Error()
}
return "lookup account " + e.Name + ": " + s
}
func (e *AccountLookupError) Unwrap() error { return e.Err }
type SddlConversionError struct {
Sddl string
Err error
}
func (e *SddlConversionError) Error() string {
return "convert " + e.Sddl + ": " + e.Err.Error()
}
func (e *SddlConversionError) Unwrap() error { return e.Err }
// LookupSidByName looks up the SID of an account by name
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupSidByName(name string) (sid string, err error) {
if name == "" {
return "", &AccountLookupError{name, windows.ERROR_NONE_MAPPED}
}
var sidSize, sidNameUse, refDomainSize uint32
err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{name, err}
}
sidBuffer := make([]byte, sidSize)
refDomainBuffer := make([]uint16, refDomainSize)
err = lookupAccountName(nil, name, &sidBuffer[0], &sidSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
if err != nil {
return "", &AccountLookupError{name, err}
}
var strBuffer *uint16
err = convertSidToStringSid(&sidBuffer[0], &strBuffer)
if err != nil {
return "", &AccountLookupError{name, err}
}
sid = windows.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(strBuffer))[:])
_, _ = windows.LocalFree(windows.Handle(unsafe.Pointer(strBuffer)))
return sid, nil
}
// LookupNameBySid looks up the name of an account by SID
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupNameBySid(sid string) (name string, err error) {
if sid == "" {
return "", &AccountLookupError{sid, windows.ERROR_NONE_MAPPED}
}
sidBuffer, err := windows.UTF16PtrFromString(sid)
if err != nil {
return "", &AccountLookupError{sid, err}
}
var sidPtr *byte
if err = convertStringSidToSid(sidBuffer, &sidPtr); err != nil {
return "", &AccountLookupError{sid, err}
}
defer windows.LocalFree(windows.Handle(unsafe.Pointer(sidPtr))) //nolint:errcheck
var nameSize, refDomainSize, sidNameUse uint32
err = lookupAccountSid(nil, sidPtr, nil, &nameSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{sid, err}
}
nameBuffer := make([]uint16, nameSize)
refDomainBuffer := make([]uint16, refDomainSize)
err = lookupAccountSid(nil, sidPtr, &nameBuffer[0], &nameSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
if err != nil {
return "", &AccountLookupError{sid, err}
}
name = windows.UTF16ToString(nameBuffer)
return name, nil
}
func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
sd, err := windows.SecurityDescriptorFromString(sddl)
if err != nil {
return nil, &SddlConversionError{Sddl: sddl, Err: err}
}
b := unsafe.Slice((*byte)(unsafe.Pointer(sd)), sd.Length())
return b, nil
}
func SecurityDescriptorToSddl(sd []byte) (string, error) {
if l := int(unsafe.Sizeof(windows.SECURITY_DESCRIPTOR{})); len(sd) < l {
return "", fmt.Errorf("SecurityDescriptor (%d) smaller than expected (%d): %w", len(sd), l, windows.ERROR_INCORRECT_SIZE)
}
s := (*windows.SECURITY_DESCRIPTOR)(unsafe.Pointer(&sd[0]))
return s.String(), nil
}

View File

@@ -0,0 +1,5 @@
//go:build windows
package winio
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./*.go

View File

@@ -0,0 +1,378 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package winio
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modadvapi32 = windows.NewLazySystemDLL("advapi32.dll")
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
modntdll = windows.NewLazySystemDLL("ntdll.dll")
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges")
procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW")
procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW")
procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf")
procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW")
procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW")
procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW")
procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW")
procOpenThreadToken = modadvapi32.NewProc("OpenThreadToken")
procRevertToSelf = modadvapi32.NewProc("RevertToSelf")
procBackupRead = modkernel32.NewProc("BackupRead")
procBackupWrite = modkernel32.NewProc("BackupWrite")
procCancelIoEx = modkernel32.NewProc("CancelIoEx")
procConnectNamedPipe = modkernel32.NewProc("ConnectNamedPipe")
procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort")
procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW")
procDisconnectNamedPipe = modkernel32.NewProc("DisconnectNamedPipe")
procGetCurrentThread = modkernel32.NewProc("GetCurrentThread")
procGetNamedPipeHandleStateW = modkernel32.NewProc("GetNamedPipeHandleStateW")
procGetNamedPipeInfo = modkernel32.NewProc("GetNamedPipeInfo")
procGetQueuedCompletionStatus = modkernel32.NewProc("GetQueuedCompletionStatus")
procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes")
procNtCreateNamedPipeFile = modntdll.NewProc("NtCreateNamedPipeFile")
procRtlDefaultNpAcl = modntdll.NewProc("RtlDefaultNpAcl")
procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult")
)
func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
var _p0 uint32
if releaseAll {
_p0 = 1
}
r0, _, e1 := syscall.SyscallN(procAdjustTokenPrivileges.Addr(), uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize)))
success = r0 != 0
if true {
err = errnoErr(e1)
}
return
}
func convertSidToStringSid(sid *byte, str **uint16) (err error) {
r1, _, e1 := syscall.SyscallN(procConvertSidToStringSidW.Addr(), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(str)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func convertStringSidToSid(str *uint16, sid **byte) (err error) {
r1, _, e1 := syscall.SyscallN(procConvertStringSidToSidW.Addr(), uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func impersonateSelf(level uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procImpersonateSelf.Addr(), uintptr(level))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(accountName)
if err != nil {
return
}
return _lookupAccountName(systemName, _p0, sid, sidSize, refDomain, refDomainSize, sidNameUse)
}
func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procLookupAccountNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procLookupAccountSidW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
if err != nil {
return
}
return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageId)
}
func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procLookupPrivilegeDisplayNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
if err != nil {
return
}
return _lookupPrivilegeName(_p0, luid, buffer, size)
}
func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procLookupPrivilegeNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
if err != nil {
return
}
var _p1 *uint16
_p1, err = syscall.UTF16PtrFromString(name)
if err != nil {
return
}
return _lookupPrivilegeValue(_p0, _p1, luid)
}
func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) {
r1, _, e1 := syscall.SyscallN(procLookupPrivilegeValueW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func openThreadToken(thread windows.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) {
var _p0 uint32
if openAsSelf {
_p0 = 1
}
r1, _, e1 := syscall.SyscallN(procOpenThreadToken.Addr(), uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func revertToSelf() (err error) {
r1, _, e1 := syscall.SyscallN(procRevertToSelf.Addr())
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func backupRead(h windows.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) {
var _p0 *byte
if len(b) > 0 {
_p0 = &b[0]
}
var _p1 uint32
if abort {
_p1 = 1
}
var _p2 uint32
if processSecurity {
_p2 = 1
}
r1, _, e1 := syscall.SyscallN(procBackupRead.Addr(), uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func backupWrite(h windows.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) {
var _p0 *byte
if len(b) > 0 {
_p0 = &b[0]
}
var _p1 uint32
if abort {
_p1 = 1
}
var _p2 uint32
if processSecurity {
_p2 = 1
}
r1, _, e1 := syscall.SyscallN(procBackupWrite.Addr(), uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func cancelIoEx(file windows.Handle, o *windows.Overlapped) (err error) {
r1, _, e1 := syscall.SyscallN(procCancelIoEx.Addr(), uintptr(file), uintptr(unsafe.Pointer(o)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func connectNamedPipe(pipe windows.Handle, o *windows.Overlapped) (err error) {
r1, _, e1 := syscall.SyscallN(procConnectNamedPipe.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(o)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func createIoCompletionPort(file windows.Handle, port windows.Handle, key uintptr, threadCount uint32) (newport windows.Handle, err error) {
r0, _, e1 := syscall.SyscallN(procCreateIoCompletionPort.Addr(), uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount))
newport = windows.Handle(r0)
if newport == 0 {
err = errnoErr(e1)
}
return
}
func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(name)
if err != nil {
return
}
return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa)
}
func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) {
r0, _, e1 := syscall.SyscallN(procCreateNamedPipeW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)))
handle = windows.Handle(r0)
if handle == windows.InvalidHandle {
err = errnoErr(e1)
}
return
}
func disconnectNamedPipe(pipe windows.Handle) (err error) {
r1, _, e1 := syscall.SyscallN(procDisconnectNamedPipe.Addr(), uintptr(pipe))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getCurrentThread() (h windows.Handle) {
r0, _, _ := syscall.SyscallN(procGetCurrentThread.Addr())
h = windows.Handle(r0)
return
}
func getNamedPipeHandleState(pipe windows.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procGetNamedPipeHandleStateW.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getNamedPipeInfo(pipe windows.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procGetNamedPipeInfo.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getQueuedCompletionStatus(port windows.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) {
r1, _, e1 := syscall.SyscallN(procGetQueuedCompletionStatus.Addr(), uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func setFileCompletionNotificationModes(h windows.Handle, flags uint8) (err error) {
r1, _, e1 := syscall.SyscallN(procSetFileCompletionNotificationModes.Addr(), uintptr(h), uintptr(flags))
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func ntCreateNamedPipeFile(pipe *windows.Handle, access ntAccessMask, oa *objectAttributes, iosb *ioStatusBlock, share ntFileShareMode, disposition ntFileCreationDisposition, options ntFileOptions, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) {
r0, _, _ := syscall.SyscallN(procNtCreateNamedPipeFile.Addr(), uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)))
status = ntStatus(r0)
return
}
func rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) {
r0, _, _ := syscall.SyscallN(procRtlDefaultNpAcl.Addr(), uintptr(unsafe.Pointer(dacl)))
status = ntStatus(r0)
return
}
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) {
r0, _, _ := syscall.SyscallN(procRtlDosPathNameToNtPathName_U.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved))
status = ntStatus(r0)
return
}
func rtlNtStatusToDosError(status ntStatus) (winerr error) {
r0, _, _ := syscall.SyscallN(procRtlNtStatusToDosErrorNoTeb.Addr(), uintptr(status))
if r0 != 0 {
winerr = syscall.Errno(r0)
}
return
}
func wsaGetOverlappedResult(h windows.Handle, o *windows.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) {
var _p0 uint32
if wait {
_p0 = 1
}
r1, _, e1 := syscall.SyscallN(procWSAGetOverlappedResult.Addr(), uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)))
if r1 == 0 {
err = errnoErr(e1)
}
return
}

View File

@@ -0,0 +1,3 @@
# This source code refers to The Go Authors for copyright purposes.
# The master list of authors is in the main Go distribution,
# visible at https://tip.golang.org/AUTHORS.

View File

@@ -0,0 +1,3 @@
# This source code was written by the Go contributors.
# The master list of contributors is in the main Go distribution,
# visible at https://tip.golang.org/CONTRIBUTORS.

View File

@@ -0,0 +1,27 @@
Copyright (c) 2020 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@@ -0,0 +1,176 @@
# JSON Serialization (v2)
[![GoDev](https://img.shields.io/static/v1?label=godev&message=reference&color=00add8)](https://pkg.go.dev/github.com/go-json-experiment/json)
[![Build Status](https://github.com/go-json-experiment/json/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/go-json-experiment/json/actions)
This module hosts an experimental implementation of v2 `encoding/json`.
The API is unstable and breaking changes will regularly be made.
Do not depend on this in publicly available modules.
Any commits that make breaking API or behavior changes will be marked
with the string "WARNING: " near the top of the commit message.
It is your responsibility to inspect the list of commit changes
when upgrading the module. Not all breaking changes will lead to build failures.
## Current status
A [proposal to include this module in Go as `encoding/json/v2` and `encoding/json/jsontext`](https://github.com/golang/go/issues/71497)
has been started on the Go Github project on 2025-01-30. Please provide your feedback there.
At present, this module is available within the Go standard library as a Go experiment.
See ["A new experimental Go API for JSON"](https://go.dev/blog/jsonv2-exp) for more details.
On 2025-11-20, a `json/v2` working group was established by the Go project
consisting of @aclements, @neild, @prattmic, @ChrisHines, and @dsnet
to review outstanding issues in preparing for formal adoption into the standard library.
The working group meets weekly and notes are recorded on [issue #76406](https://github.com/golang/go/issues/76406).
See [the project dashboard](https://github.com/orgs/golang/projects/50/views/1)
to get a sense of what issues remain for the meeting group to resolve.
The "Under review" and "Needs review" categories need to be emptied
before `json/v2` can be released.
This repository will continue to import and mirror the upstream Go project
for the foreseeable future. Consequently, code changes should be made at
the upstream [Go project](https://go.googlesource.com/go/) instead of here.
Building this module with the `goexperiment.jsonv2` tag will cause the
module to use the `encoding/json/v2` package under-the-hood,
otherwise this will provide a different implementation of `json/v2`.
## Goals and objectives
* **Mostly backwards compatible:** If possible, v2 should aim to be _mostly_
compatible with v1 in terms of both API and default behavior to ease migration.
For example, the `Marshal` and `Unmarshal` functions are the most widely used
declarations in the v1 package. It seems sensible for equivalent functionality
in v2 to be named the same and have a mostly compatible signature.
Behaviorally, we should aim for 95% to 99% backwards compatibility.
We do not aim for 100% compatibility since we want the freedom to break
certain behaviors that are now considered to have been a mistake.
Options exist that can bring the v2 implementation to 100% compatibility,
but it will not be the default.
* **More flexible:** There is a
[long list of feature requests](https://github.com/golang/go/issues?q=is%3Aissue+is%3Aopen+encoding%2Fjson+in%3Atitle).
We should aim to provide the most flexible features that addresses most usages.
We do not want to over fit the v2 API to handle every possible use case.
Ideally, the features provided should be orthogonal in nature such that
any combination of features results in as few surprising edge cases as possible.
* **More performant:** JSON serialization is widely used and any bit of extra
performance gains will be greatly appreciated. Some rarely used behaviors of v1
may be dropped in favor of better performance. For example,
despite `Encoder` and `Decoder` operating on an `io.Writer` and `io.Reader`,
they do not operate in a truly streaming manner,
leading to a loss in performance. The v2 implementation should aim to be truly
streaming by default (see [#33714](https://golang.org/issue/33714)).
* **Easy to use (hard to misuse):** The v2 API should aim to make
the common case easy and the less common case at least possible.
The API should avoid behavior that goes contrary to user expectation,
which may result in subtle bugs (see [#36225](https://golang.org/issue/36225)).
* **v1 and v2 maintainability:** Since the v1 implementation must stay forever,
it would be beneficial if v1 could be implemented under the hood with v2,
allowing for less maintenance burden in the future. This probably implies that
behavioral changes in v2 relative to v1 need to be exposed as options.
* **Avoid unsafe:** Standard library packages generally avoid the use of
package `unsafe` even if it could provide a performance boost.
We aim to preserve this property.
## Development
This module is primarily developed by
[@dsnet](https://github.com/dsnet),
[@mvdan](https://github.com/mvdan), and
[@johanbrandhorst](https://github.com/johanbrandhorst)
with feedback provided by
[@rogpeppe](https://github.com/rogpeppe),
[@ChrisHines](https://github.com/ChrisHines), and
[@rsc](https://github.com/rsc).
Discussion about semantics occur semi-regularly, where a
[record of past meetings can be found here](https://docs.google.com/document/d/1rovrOTd-wTawGMPPlPuKhwXaYBg9VszTXR9AQQL5LfI/edit?usp=sharing).
## Design overview
This package aims to provide a clean separation between syntax and semantics.
Syntax deals with the structural representation of JSON (as specified in
[RFC 4627](https://tools.ietf.org/html/rfc4627),
[RFC 7159](https://tools.ietf.org/html/rfc7159),
[RFC 7493](https://tools.ietf.org/html/rfc7493),
[RFC 8259](https://tools.ietf.org/html/rfc8259), and
[RFC 8785](https://tools.ietf.org/html/rfc8785)).
Semantics deals with the meaning of syntactic data as usable application data.
The `Encoder` and `Decoder` types are streaming tokenizers concerned with the
packing or parsing of JSON data. They operate on `Token` and `Value` types
which represent the common data structures that are representable in JSON.
`Encoder` and `Decoder` do not aim to provide any interpretation of the data.
Functions like `Marshal`, `MarshalWrite`, `MarshalEncode`, `Unmarshal`,
`UnmarshalRead`, and `UnmarshalDecode` provide semantic meaning by correlating
any arbitrary Go type with some JSON representation of that type (as stored in
data types like `[]byte`, `io.Writer`, `io.Reader`, `Encoder`, or `Decoder`).
![API overview](api.png)
This diagram provides a high-level overview of the v2 `json` and `jsontext` packages.
Purple blocks represent types, while blue blocks represent functions or methods.
The arrows and their direction represent the approximate flow of data.
The bottom half of the diagram contains functionality that is only concerned
with syntax (implemented by the `jsontext` package),
while the upper half contains functionality that assigns
semantic meaning to syntactic data handled by the bottom half
(as implemented by the v2 `json` package).
In contrast to v1 `encoding/json`, options are represented as separate types
rather than being setter methods on the `Encoder` or `Decoder` types.
Some options affects JSON serialization at the syntactic layer,
while others affect it at the semantic layer.
Some options only affect JSON when decoding,
while others affect JSON while encoding.
## Behavior changes
The v2 `json` package changes the default behavior of `Marshal` and `Unmarshal`
relative to the v1 `json` package to be more sensible.
Some of these behavior changes have options and workarounds to opt into
behavior similar to what v1 provided.
This table shows an overview of the changes:
| v1 | v2 | Details |
| -- | -- | ------- |
| JSON object members are unmarshaled into a Go struct using a **case-insensitive name match**. | JSON object members are unmarshaled into a Go struct using a **case-sensitive name match**. | [CaseSensitivity](/v1/diff_test.go#:~:text=TestCaseSensitivity) |
| When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value is an empty Go value**, which is defined as false, 0, a nil pointer, a nil interface value, and any empty array, slice, map, or string. | When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value would encode as an empty JSON value**, which is defined as a JSON null, or an empty JSON string, object, or array. | [OmitEmptyOption](/v1/diff_test.go#:~:text=TestOmitEmptyOption) |
| The `string` option **does affect** Go strings and bools. | The `string` option **does not affect** Go strings or bools. | [StringOption](/v1/diff_test.go#:~:text=TestStringOption) |
| The `string` option **does not recursively affect** sub-values of the Go field value. | The `string` option **does recursively affect** sub-values of the Go field value. | [StringOption](/v1/diff_test.go#:~:text=TestStringOption) |
| The `string` option **sometimes accepts** a JSON null escaped within a JSON string. | The `string` option **never accepts** a JSON null escaped within a JSON string. | [StringOption](/v1/diff_test.go#:~:text=TestStringOption) |
| A nil Go slice is marshaled as a **JSON null**. | A nil Go slice is marshaled as an **empty JSON array**. | [NilSlicesAndMaps](/v1/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A nil Go map is marshaled as a **JSON null**. | A nil Go map is marshaled as an **empty JSON object**. | [NilSlicesAndMaps](/v1/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A Go array may be unmarshaled from a **JSON array of any length**. | A Go array must be unmarshaled from a **JSON array of the same length**. | [Arrays](/v1/diff_test.go#:~:text=Arrays) |
| A Go byte array is represented as a **JSON array of JSON numbers**. | A Go byte array is represented as a **Base64-encoded JSON string**. | [ByteArrays](/v1/diff_test.go#:~:text=TestByteArrays) |
| `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **inconsistently called**. | `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **consistently called**. | [PointerReceiver](/v1/diff_test.go#:~:text=TestPointerReceiver) |
| A Go map is marshaled in a **deterministic order**. | A Go map is marshaled in a **non-deterministic order**. | [MapDeterminism](/v1/diff_test.go#:~:text=TestMapDeterminism) |
| JSON strings are encoded **with HTML-specific characters being escaped**. | JSON strings are encoded **without any characters being escaped** (unless necessary). | [EscapeHTML](/v1/diff_test.go#:~:text=TestEscapeHTML) |
| When marshaling, invalid UTF-8 within a Go string **are silently replaced**. | When marshaling, invalid UTF-8 within a Go string **results in an error**. | [InvalidUTF8](/v1/diff_test.go#:~:text=TestInvalidUTF8) |
| When unmarshaling, invalid UTF-8 within a JSON string **are silently replaced**. | When unmarshaling, invalid UTF-8 within a JSON string **results in an error**. | [InvalidUTF8](/v1/diff_test.go#:~:text=TestInvalidUTF8) |
| When marshaling, **an error does not occur** if the output JSON value contains objects with duplicate names. | When marshaling, **an error does occur** if the output JSON value contains objects with duplicate names. | [DuplicateNames](/v1/diff_test.go#:~:text=TestDuplicateNames) |
| When unmarshaling, **an error does not occur** if the input JSON value contains objects with duplicate names. | When unmarshaling, **an error does occur** if the input JSON value contains objects with duplicate names. | [DuplicateNames](/v1/diff_test.go#:~:text=TestDuplicateNames) |
| Unmarshaling a JSON null into a non-empty Go value **inconsistently clears the value or does nothing**. | Unmarshaling a JSON null into a non-empty Go value **always clears the value**. | [MergeNull](/v1/diff_test.go#:~:text=TestMergeNull) |
| Unmarshaling a JSON value into a non-empty Go value **follows inconsistent and bizarre behavior**. | Unmarshaling a JSON value into a non-empty Go value **always merges if the input is an object, and otherwise replaces**. | [MergeComposite](/v1/diff_test.go#:~:text=TestMergeComposite) |
| A `time.Duration` is represented as a **JSON number containing the decimal number of nanoseconds**. | A `time.Duration` has no default representation in v2 (see [#71631](https://golang.org/issue/71631)) and results in an error. | |
| A Go struct with only unexported fields **can be serialized**. | A Go struct with only unexported fields **cannot be serialized**. | [EmptyStructs](/v1/diff_test.go#:~:text=TestEmptyStructs) |
See [diff_test.go](/v1/diff_test.go) for details about every change.
## Performance
One of the goals of the v2 module is to be more performant than v1,
but not at the expense of correctness.
In general, v2 is at performance parity with v1 for marshaling,
but dramatically faster for unmarshaling.
See https://github.com/go-json-experiment/jsonbench for benchmarks
comparing v2 with v1 and a number of other popular JSON implementations.

View File

@@ -0,0 +1,943 @@
// Copyright 2025 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated by alias_gen.go; DO NOT EDIT.
//go:build goexperiment.jsonv2 && go1.25
// Package json implements semantic processing of JSON as specified in RFC 8259.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
// [Marshal] and [Unmarshal] encode and decode Go values
// to/from JSON text contained within a []byte.
// [MarshalWrite] and [UnmarshalRead] operate on JSON text
// by writing to or reading from an [io.Writer] or [io.Reader].
// [MarshalEncode] and [UnmarshalDecode] operate on JSON text
// by encoding to or decoding from a [jsontext.Encoder] or [jsontext.Decoder].
// [Options] may be passed to each of the marshal or unmarshal functions
// to configure the semantic behavior of marshaling and unmarshaling
// (i.e., alter how JSON data is understood as Go data and vice versa).
// [jsontext.Options] may also be passed to the marshal or unmarshal functions
// to configure the syntactic behavior of encoding or decoding.
//
// The data types of JSON are mapped to/from the data types of Go based on
// the closest logical equivalent between the two type systems. For example,
// a JSON boolean corresponds with a Go bool,
// a JSON string corresponds with a Go string,
// a JSON number corresponds with a Go int, uint or float,
// a JSON array corresponds with a Go slice or array, and
// a JSON object corresponds with a Go struct or map.
// See the documentation on [Marshal] and [Unmarshal] for a comprehensive list
// of how the JSON and Go type systems correspond.
//
// Arbitrary Go types can customize their JSON representation by implementing
// [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom].
// This provides authors of Go types with control over how their types are
// serialized as JSON. Alternatively, users can implement functions that match
// [MarshalFunc], [MarshalToFunc], [UnmarshalFunc], or [UnmarshalFromFunc]
// to specify the JSON representation for arbitrary types.
// This provides callers of JSON functionality with control over
// how any arbitrary type is serialized as JSON.
//
// # JSON Representation of Go structs
//
// A Go struct is naturally represented as a JSON object,
// where each Go struct field corresponds with a JSON object member.
// When marshaling, all Go struct fields are recursively encoded in depth-first
// order as JSON object members except those that are ignored or omitted.
// When unmarshaling, JSON object members are recursively decoded
// into the corresponding Go struct fields.
// Object members that do not match any struct fields,
// also known as “unknown members”, are ignored by default or rejected
// if [RejectUnknownMembers] is specified.
//
// The representation of each struct field can be customized in the
// "json" struct field tag, where the tag is a comma separated list of options.
// As a special case, if the entire tag is `json:"-"`,
// then the field is ignored with regard to its JSON representation.
// Some options also have equivalent behavior controlled by a caller-specified [Options].
// Field-specified options take precedence over caller-specified options.
//
// The first option is the JSON object name override for the Go struct field.
// If the name is not specified, then the Go struct field name
// is used as the JSON object name.
// By default, unmarshaling uses case-sensitive matching to identify
// the Go struct field associated with a JSON object name.
//
// After the name, the following tag options are supported:
//
// - omitzero: When marshaling, the "omitzero" option specifies that
// the struct field should be omitted if the field value is zero
// as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This option has no effect when unmarshaling.
//
// - omitempty: When marshaling, the "omitempty" option specifies that
// the struct field should be omitted if the field value would have been
// encoded as a JSON null, empty string, empty object, or empty array.
// This option has no effect when unmarshaling.
//
// - string: The "string" option specifies that [StringifyNumbers] be set
// when marshaling or unmarshaling a struct field value.
// This causes types that would normally be encoded as a JSON number
// to instead be encoded as a JSON number quoted within a JSON string,
// and to be decoded from a JSON string containing the JSON number
// without any surrounding whitespace.
// The "string" option only applies to the top-level of the Go struct field value.
// Specifically, for the default representation of composite Go data types
// (e.g., array, slice, struct, or map), it will not stringify JSON numbers
// within such types. Applying this option to invalid types causes a runtime error.
// This extra level of encoding is often necessary since many JSON parsers
// cannot precisely represent 64-bit integers.
//
// - case: When unmarshaling, the "case" option specifies how
// JSON object names are matched with the JSON name for Go struct fields.
// The option is a key-value pair specified as "case:value" where
// the value must either be 'ignore' or 'strict'.
// The 'ignore' value specifies that matching is case-insensitive
// where dashes and underscores are also ignored. If multiple fields match,
// then the field with an exact name match is selected, otherwise an error
// is reported due to an ambiguous set of candidate fields to unmarshal into.
// The 'strict' value specifies that matching is case-sensitive.
// This takes precedence over the [MatchCaseInsensitiveNames] option.
//
// - embed: The "embed" option specifies that
// the JSON representable content of this field type is to be promoted
// as if they were specified in the parent struct.
// It is the JSON equivalent of Go struct embedding.
// A Go embedded field is implicitly JSON embedded unless
// an explicit JSON name is specified. The embedded field must be a Go struct
// (that does not implement any JSON methods), [jsontext.Value],
// map[~string]T, or an unnamed pointer to such types. When marshaling,
// embedded fields from a pointer type are omitted if it is nil.
// Embedded fields of type [jsontext.Value] and map[~string]T are called
// “embedded fallbacks” as they can represent all possible
// JSON object members not directly handled by the parent struct.
// Only one embedded fallback field may be specified in a struct,
// while many non-fallback fields may be specified. This option
// must not be specified with any other option (including the JSON name).
//
// The "omitzero" and "omitempty" options are mostly semantically identical.
// The former is defined in terms of the Go type system,
// while the latter in terms of the JSON type system.
// Consequently they behave differently in some circumstances.
// For example, only a nil slice or map is omitted under "omitzero", while
// an empty slice or map is omitted under "omitempty" regardless of nilness.
// The "omitzero" option is useful for types with a well-defined zero value
// (e.g., [net/netip.Addr]) or have an IsZero method (e.g., [time.Time.IsZero]).
//
// Every Go struct corresponds to a list of JSON representable fields
// which is constructed by performing a breadth-first search over
// all struct fields (excluding unexported or ignored fields),
// where the search recursively descends into embedded structs.
// The set of non-embedded fields in a struct must have unique JSON names.
// If multiple fields all have the same JSON name, then the one
// at shallowest depth takes precedence and the other fields at deeper depths
// are excluded from the list of JSON representable fields.
// If multiple fields at the shallowest depth have the same JSON name,
// but exactly one is explicitly tagged with a JSON name,
// then that field takes precedence and all others are excluded from the list.
// This is analogous to Go visibility rules for struct field selection
// with embedded struct types.
//
// Marshaling or unmarshaling a non-empty struct
// without any JSON representable fields results in a [SemanticError].
// Unexported fields must not have any `json` tags except for `json:"-"`.
//
// # Security Considerations
//
// JSON is frequently used as a data interchange format to communicate
// between different systems, possibly implemented in different languages.
// For interoperability and security reasons, it is important that
// all implementations agree upon the semantic meaning of the data.
//
// [For example, suppose we have two micro-services.]
// The first service is responsible for authenticating a JSON request,
// while the second service is responsible for executing the request
// (having assumed that the prior service authenticated the request).
// If an attacker were able to maliciously craft a JSON request such that
// both services believe that the same request is from different users,
// it could bypass the authenticator with valid credentials for one user,
// but maliciously perform an action on behalf of a different user.
//
// According to RFC 8259, there unfortunately exist many JSON texts
// that are syntactically valid but semantically ambiguous.
// For example, the standard does not define how to interpret duplicate
// names within an object.
//
// The v1 [encoding/json] and [encoding/json/v2] packages
// interpret some inputs in different ways. In particular:
//
// - The standard specifies that JSON must be encoded using UTF-8.
// By default, v1 replaces invalid bytes of UTF-8 in JSON strings
// with the Unicode replacement character,
// while v2 rejects inputs with invalid UTF-8.
// To change the default, specify the [jsontext.AllowInvalidUTF8] option.
// The replacement of invalid UTF-8 is a form of data corruption
// that alters the precise meaning of strings.
//
// - The standard does not specify a particular behavior when
// duplicate names are encountered within a JSON object,
// which means that different implementations may behave differently.
// By default, v1 allows for the presence of duplicate names,
// while v2 rejects duplicate names.
// To change the default, specify the [jsontext.AllowDuplicateNames] option.
// If allowed, object members are processed in the order they are observed,
// meaning that later values will replace or be merged into prior values,
// depending on the Go value type.
//
// - The standard defines a JSON object as an unordered collection of name/value pairs.
// While ordering can be observed through the underlying [jsontext] API,
// both v1 and v2 generally avoid exposing the ordering.
// No application should semantically depend on the order of object members.
// Allowing duplicate names is a vector through which ordering of members
// can accidentally be observed and depended upon.
//
// - The standard suggests that JSON object names are typically compared
// based on equality of the sequence of Unicode code points,
// which implies that comparing names is often case-sensitive.
// When unmarshaling a JSON object into a Go struct,
// by default, v1 uses a (loose) case-insensitive match on the name,
// while v2 uses a (strict) case-sensitive match on the name.
// To change the default, specify the [MatchCaseInsensitiveNames] option.
// The use of case-insensitive matching provides another vector through
// which duplicate names can occur. Allowing case-insensitive matching
// means that v1 or v2 might interpret JSON objects differently from most
// other JSON implementations (which typically use a case-sensitive match).
//
// - The standard does not specify a particular behavior when
// an unknown name in a JSON object is encountered.
// When unmarshaling a JSON object into a Go struct, by default
// both v1 and v2 ignore unknown names and their corresponding values.
// To change the default, specify the [RejectUnknownMembers] option.
//
// - The standard suggests that implementations may use a float64
// to represent a JSON number. Consequently, large JSON integers
// may lose precision when stored as a floating-point type.
// Both v1 and v2 correctly preserve precision when marshaling and
// unmarshaling a concrete integer type. However, even if v1 and v2
// preserve precision for concrete types, other JSON implementations
// may not be able to preserve precision for outputs produced by v1 or v2.
// The `string` tag option can be used to specify that an integer type
// is to be quoted within a JSON string to avoid loss of precision.
// Furthermore, v1 and v2 may still lose precision when unmarshaling
// into an any interface value, where unmarshal uses a float64
// by default to represent a JSON number.
// To change the default, specify the [WithUnmarshalers] option
// with a custom unmarshaler that pre-populates the interface value
// with a concrete Go type that can preserve precision.
//
// RFC 8785 specifies a canonical form for any JSON text,
// which explicitly defines specific behaviors that RFC 8259 leaves undefined.
// In theory, if a text can successfully [jsontext.Value.Canonicalize]
// without changing the semantic meaning of the data, then it provides a
// greater degree of confidence that the data is more secure and interoperable.
//
// The v2 API generally chooses more secure defaults than v1,
// but care should still be taken with large integers or unknown members.
//
// [For example, suppose we have two micro-services.]: https://www.youtube.com/watch?v=avilmOcHKHE&t=1057s
package json
import (
"encoding/json/jsontext"
"encoding/json/v2"
"io"
)
// Marshal serializes a Go value as a []byte according to the provided
// marshal and encode options (while ignoring unmarshal or decode options).
// It does not terminate the output with a newline.
//
// Type-specific marshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when encoding
// a value of type T (by taking its address) or a non-nil value of *T.
// Marshal ensures that a value is always addressable
// (by copying the value if necessary) so that
// these functions and methods can be consistently called. For performance,
// it is recommended that Marshal be passed a non-nil pointer to the value.
//
// The input value is encoded as JSON according to the following rules:
//
// - If any type-specific functions in a [WithMarshalers] option match
// the value type, then those functions are called to encode the value.
// If all applicable functions return [errors.ErrUnsupported],
// then the value is encoded according to subsequent rules.
//
// - If the value type implements [MarshalerTo],
// then the MarshalJSONTo method is called to encode the value.
// If the method returns [errors.ErrUnsupported],
// then the input is encoded according to subsequent rules.
//
// - If the value type implements [Marshaler],
// then the MarshalJSON method is called to encode the value.
//
// - If the value type implements [encoding.TextAppender],
// then the AppendText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - If the value type implements [encoding.TextMarshaler],
// then the MarshalText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - Otherwise, the value is encoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation as follows:
//
// - A Go boolean is encoded as a JSON boolean (e.g., true or false).
//
// - A Go string is encoded as a JSON string.
//
// - A Go []byte or [N]byte is encoded as a JSON string containing
// a binary value using Base 64 Encoding per RFC 4648, section 4.
//
// - A Go integer is encoded as a JSON number without fractions or exponents.
// If [StringifyNumbers] is specified or encoding a JSON object name,
// then the JSON number is encoded within a JSON string.
//
// - A Go float is encoded as a JSON number.
// If [StringifyNumbers] is specified or encoding a JSON object name,
// then the JSON number is encoded within a JSON string.
// Encoding a NaN or ±Inf results in a [SemanticError].
//
// - A Go map is encoded as a JSON object, where each Go map key and value
// is recursively encoded as a name and value pair in the JSON object.
// The Go map key must encode as a JSON string, otherwise this results
// in a [SemanticError]. The Go map is traversed in a non-deterministic order.
// For deterministic encoding, consider using the [Deterministic] option.
// By default, a nil map is encoded as an empty JSON object,
// unless the [FormatNilMapAsNull] option is specified.
//
// - A Go struct is encoded as a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is encoded as a JSON array, where each Go slice element
// is recursively JSON-encoded as the elements of the JSON array.
// By default, a nil slice is encoded as an empty JSON array,
// unless the [FormatNilSliceAsNull] option is specified.
//
// - A Go array is encoded as a JSON array, where each Go array element
// is recursively JSON-encoded as the elements of the JSON array.
// The JSON array length is always identical to the Go array length.
//
// - A Go pointer is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
//
// - A Go interface is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
//
// - A Go [time.Time] is encoded as a JSON string containing the timestamp
// formatted in RFC 3339 with nanosecond precision.
//
// - A Go [time.Duration] currently has no default representation and
// results in a [SemanticError], unless the [encoding/json.FormatDurationAsNano]
// option is specified, in which case it is encoded as a JSON number
// without fractions or exponents, representing the duration in nanoseconds.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a [SemanticError].
//
// JSON cannot represent cyclic data structures and Marshal does not handle them.
// Passing cyclic structures will result in an error.
func Marshal(in any, opts ...Options) (out []byte, err error) {
opts = mayAppendSupportFormatTag(opts)
return json.Marshal(in, opts...)
}
// MarshalWrite serializes a Go value into an [io.Writer] according to the provided
// marshal and encode options (while ignoring unmarshal or decode options).
// It does not terminate the output with a newline.
// See [Marshal] for details about the conversion of a Go value into JSON.
func MarshalWrite(out io.Writer, in any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
return json.MarshalWrite(out, in, opts...)
}
// MarshalEncode serializes a Go value into an [jsontext.Encoder] according to
// the provided marshal or encode options (while ignoring unmarshal or decode options).
// The options provided take precedence over options already applied on
// the [jsontext.Encoder] and only apply for the duration of the marshal call.
//
// See [Marshal] for details about the conversion of a Go value into JSON.
func MarshalEncode(out *jsontext.Encoder, in any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
return json.MarshalEncode(out, in, opts...)
}
// Unmarshal decodes a []byte input into a Go value according to the provided
// unmarshal and decode options (while ignoring marshal or encode options).
// The input must be a single JSON value with optional whitespace interspersed.
// The output must be a non-nil pointer.
//
// Type-specific unmarshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when decoding
// a value of type T (by taking its address) or a non-nil value of *T.
// Unmarshal ensures that a value is always addressable
// (by copying the value if necessary) so that
// these functions and methods can be consistently called.
// If a value must be shallow copied to call a pointer-receiver
// [Unmarshaler], [UnmarshalerFrom], or [encoding.TextUnmarshaler] method,
// then any mutations performed by the method are shallow copied back
// into the destination value.
//
// The input is decoded into the output according to the following rules:
//
// - If any type-specific functions in a [WithUnmarshalers] option match
// the value type, then those functions are called to decode the JSON
// value. If all applicable functions return [errors.ErrUnsupported],
// then the input is decoded according to subsequent rules.
//
// - If the value type implements [UnmarshalerFrom],
// then the UnmarshalJSONFrom method is called to decode the JSON value.
// If the method returns [errors.ErrUnsupported],
// then the input is decoded according to subsequent rules.
//
// - If the value type implements [Unmarshaler],
// then the UnmarshalJSON method is called to decode the JSON value.
//
// - If the value type implements [encoding.TextUnmarshaler],
// then the input is decoded as a JSON string and
// the UnmarshalText method is called with the decoded string value.
// This fails with a [SemanticError] if the input is not a JSON string.
//
// - Otherwise, the JSON value is decoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// A JSON null may be decoded into every supported Go value where
// it is equivalent to storing the zero value of the Go value.
// If the input JSON kind is not handled by the current Go value type,
// then this fails with a [SemanticError]. Unless otherwise specified,
// the decoded value replaces any pre-existing value.
//
// The representation of each type is as follows:
//
// - A Go boolean is decoded from a JSON boolean (e.g., true or false).
//
// - A Go string is decoded from a JSON string.
//
// - A Go []byte or [N]byte is decoded from a JSON string containing
// a binary value using Base 64 Encoding per RFC 4648, section 4.
// When decoding into a non-nil []byte, the slice length is reset to zero
// and the decoded input is appended to it.
// When decoding into a [N]byte, the input must decode to exactly N bytes,
// otherwise it fails with a [SemanticError].
//
// - A Go integer is decoded from a JSON number.
// It must be decoded from a JSON string containing a JSON number
// if [StringifyNumbers] is specified or decoding a JSON object name.
// It fails with a [SemanticError] if the JSON number
// has a fractional or exponent component.
// It also fails if it overflows the representation of the Go integer type.
//
// - A Go float is decoded from a JSON number.
// It must be decoded from a JSON string containing a JSON number
// if [StringifyNumbers] is specified or decoding a JSON object name.
// It fails if it overflows the representation of the Go float type.
// Since JSON lacks a native representation for a NaN or ±Inf,
// such values cannot be the result of decoding.
//
// - A Go map is decoded from a JSON object,
// where each JSON object name and value pair is recursively decoded
// as the Go map key and value. Maps are not cleared.
// If the Go map is nil, then a new map is allocated to decode into.
// If the decoded key matches an existing Go map entry, the entry value
// is reused by decoding the JSON object value into it.
//
// - A Go struct is decoded from a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is decoded from a JSON array, where each JSON element
// is recursively decoded and appended to the Go slice.
// Before appending into a Go slice, a new slice is allocated if it is nil,
// otherwise the slice length is reset to zero.
//
// - A Go array is decoded from a JSON array, where each JSON array element
// is recursively decoded as each corresponding Go array element.
// Each Go array element is zeroed before decoding into it.
// It fails with a [SemanticError] if the JSON array does not contain
// the exact same number of elements as the Go array.
//
// - A Go pointer is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil pointer.
// Otherwise, it allocates a new underlying value if the pointer is nil,
// and recursively JSON decodes into the underlying value.
//
// - A Go interface is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil interface value.
// Otherwise, a nil interface value of an empty interface type is initialized
// with a zero Go bool, string, float64, map[string]any, or []any if the
// input is a JSON boolean, string, number, object, or array, respectively.
// If the interface value is still nil, then this fails with a [SemanticError]
// since decoding could not determine an appropriate Go type to decode into.
// For example, unmarshaling into a nil io.Reader fails since
// there is no concrete type to populate the interface value with.
// Otherwise an underlying value exists and it recursively decodes
// the JSON input into it.
//
// - A Go [time.Time] is decoded from a JSON string containing the time
// formatted in RFC 3339 with nanosecond precision.
//
// - A Go [time.Duration] currently has no default representation and
// results in a [SemanticError], unless the [encoding/json.FormatDurationAsNano]
// option is specified, in which case it is decoded as a JSON number
// without fractions or exponents, representing the duration in nanoseconds.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a [SemanticError].
//
// In general, unmarshaling follows merge semantics (similar to RFC 7396)
// where the decoded Go value replaces the destination value
// for any JSON kind other than an object.
// For JSON objects, the input object is merged into the destination value
// where matching object members recursively apply merge semantics.
func Unmarshal(in []byte, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
return json.Unmarshal(in, out, opts...)
}
// UnmarshalRead deserializes a Go value from an [io.Reader] according to the
// provided unmarshal and decode options (while ignoring marshal or encode options).
// The input must be a single JSON value with optional whitespace interspersed.
// It consumes the entirety of [io.Reader] until [io.EOF] is encountered,
// without reporting an error for EOF. The output must be a non-nil pointer.
// See [Unmarshal] for details about the conversion of JSON into a Go value.
func UnmarshalRead(in io.Reader, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
return json.UnmarshalRead(in, out, opts...)
}
// UnmarshalDecode deserializes a Go value from a [jsontext.Decoder] according to
// the provided unmarshal or decode options (while ignoring marshal or encode options).
// The options provided take precedence over options already applied on
// the [jsontext.Decoder] and only apply for the duration of the unmarshal call.
//
// The input may be a stream of zero or more JSON values,
// where this only unmarshals the next JSON value in the stream.
// If there are no more top-level JSON values, it reports [io.EOF].
// The output must be a non-nil pointer.
// See [Unmarshal] for details about the conversion of JSON into a Go value.
func UnmarshalDecode(in *jsontext.Decoder, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
return json.UnmarshalDecode(in, out, opts...)
}
// Marshalers is a list of functions that may override the marshal behavior
// of specific types. Populate [WithMarshalers] to use it with
// [Marshal], [MarshalWrite], or [MarshalEncode].
// A nil *Marshalers is equivalent to an empty list.
// There are no exported fields or methods on Marshalers.
type Marshalers = json.Marshalers
// JoinMarshalers constructs a flattened list of marshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns [errors.ErrUnsupported],
// then the next applicable function is called,
// otherwise the default marshaling behavior is used.
//
// For example:
//
// m1 := JoinMarshalers(f1, f2)
// m2 := JoinMarshalers(f0, m1, f3) // equivalent to m3
// m3 := JoinMarshalers(f0, f1, f2, f3) // equivalent to m2
func JoinMarshalers(ms ...*Marshalers) *Marshalers {
return json.JoinMarshalers(ms...)
}
// Unmarshalers is a list of functions that may override the unmarshal behavior
// of specific types. Populate [WithUnmarshalers] to use it with
// [Unmarshal], [UnmarshalRead], or [UnmarshalDecode].
// A nil *Unmarshalers is equivalent to an empty list.
// There are no exported fields or methods on Unmarshalers.
type Unmarshalers = json.Unmarshalers
// JoinUnmarshalers constructs a flattened list of unmarshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns [errors.ErrUnsupported],
// then the next applicable function is called,
// otherwise the default unmarshaling behavior is used.
//
// For example:
//
// u1 := JoinUnmarshalers(f1, f2)
// u2 := JoinUnmarshalers(f0, u1, f3) // equivalent to u3
// u3 := JoinUnmarshalers(f0, f1, f2, f3) // equivalent to u2
func JoinUnmarshalers(us ...*Unmarshalers) *Unmarshalers {
return json.JoinUnmarshalers(us...)
}
// MarshalFunc constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value.
// The value of T must not be retained outside the function call.
// It is recommended that fn return a []byte buffer that is safe
// for the caller to retain and potentially mutate.
// It may not return [errors.ErrUnsupported].
func MarshalFunc[T any](fn func(T) ([]byte, error)) *Marshalers {
return json.MarshalFunc[T](fn)
}
// MarshalToFunc constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value by calling write methods
// on the provided encoder. It may return [errors.ErrUnsupported] such that marshaling can
// move on to the next marshal function. However, no mutable method calls may
// be called on the encoder if [errors.ErrUnsupported] is returned.
// The pointer to [jsontext.Encoder] and the value of T
// must not be retained outside the function call.
func MarshalToFunc[T any](fn func(*jsontext.Encoder, T) error) *Marshalers {
return json.MarshalToFunc[T](fn)
}
// UnmarshalFunc constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value.
// The input []byte must not be mutated.
// The input []byte and value T must not be retained outside the function call.
// It may not return [errors.ErrUnsupported].
func UnmarshalFunc[T any](fn func([]byte, T) error) *Unmarshalers {
return json.UnmarshalFunc[T](fn)
}
// UnmarshalFromFunc constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value by calling read methods
// on the provided decoder. It may return [errors.ErrUnsupported] such that unmarshaling can
// move on to the next unmarshal function. However, no mutable method calls may
// be called on the decoder if [errors.ErrUnsupported] is returned.
// The pointer to [jsontext.Decoder] and the value of T
// must not be retained outside the function call.
func UnmarshalFromFunc[T any](fn func(*jsontext.Decoder, T) error) *Unmarshalers {
return json.UnmarshalFromFunc[T](fn)
}
// Marshaler is implemented by types that can marshal themselves.
// It is recommended that types implement [MarshalerTo] unless the implementation
// is trying to avoid a hard dependency on the "jsontext" package.
//
// It is recommended that implementations return a buffer that is safe
// for the caller to retain and potentially mutate.
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError].
type Marshaler = json.Marshaler
// MarshalerTo is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerTo instead of [Marshaler]
// since this is both more performant and flexible.
// If a type implements both Marshaler and MarshalerTo,
// then MarshalerTo takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default marshal options.
//
// The implementation must write only one JSON value to the Encoder.
// Alternatively, it may return [errors.ErrUnsupported] without mutating
// the Encoder. The "json" package calling the method will
// use the next available JSON representation for the receiver type.
// Implementations must not retain the pointer to [jsontext.Encoder].
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError],
// unless it is an IO error.
//
// The MarshalJSONTo method should not be directly called as it may
// return sentinel errors that need special handling.
// Users should instead call [MarshalEncode], which handles such cases.
type MarshalerTo = json.MarshalerTo
// Unmarshaler is implemented by types that can unmarshal themselves.
// It is recommended that types implement [UnmarshalerFrom] unless the implementation
// is trying to avoid a hard dependency on the "jsontext" package.
//
// The input can be assumed to be a valid encoding of a JSON value
// if called from unmarshal functionality in this package.
// UnmarshalJSON must copy the JSON data if it is retained after returning.
// It is recommended that UnmarshalJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain or mutate the input []byte.
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError].
type Unmarshaler = json.Unmarshaler
// UnmarshalerFrom is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerFrom instead of [Unmarshaler]
// since this is both more performant and flexible.
// If a type implements both Unmarshaler and UnmarshalerFrom,
// then UnmarshalerFrom takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default unmarshal options.
//
// The implementation must read only one JSON value from the Decoder.
// It is recommended that UnmarshalJSONFrom implement merge semantics when
// unmarshaling into a pre-populated value.
// Alternatively, it may return [errors.ErrUnsupported] without mutating
// the Decoder. The "json" package calling the method will
// use the next available JSON representation for the receiver type.
// Implementations must not retain the pointer to [jsontext.Decoder].
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError],
// unless it is a [jsontext.SyntacticError] or an IO error.
//
// The UnmarshalJSONFrom method should not be directly called as it may
// return sentinel errors that need special handling.
// Users should instead call [UnmarshalDecode], which handles such cases.
type UnmarshalerFrom = json.UnmarshalerFrom
// ErrUnknownName indicates that a JSON object member could not be
// unmarshaled because the name is not known to the target Go struct.
// This error is directly wrapped within a [SemanticError] when produced.
//
// The name of an unknown JSON object member can be extracted as:
//
// err := ...
// serr, ok := errors.AsType[*json.SemanticError](err)
// if ok && serr.Err == json.ErrUnknownName {
// ptr := serr.JSONPointer // JSON pointer to unknown name
// name := ptr.LastToken() // unknown name itself
// ...
// }
//
// This error is only returned if [RejectUnknownMembers] is true.
var ErrUnknownName = json.ErrUnknownName
// SemanticError describes an error determining the meaning
// of JSON data as Go data or vice-versa.
//
// If a [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom] method
// returns a SemanticError when called by the [json] package,
// then the ByteOffset, JSONPointer, and GoType fields are automatically
// populated by the calling context if they are the zero value.
//
// The contents of this error as produced by this package may change over time.
type SemanticError = json.SemanticError
// Options configure [Marshal], [MarshalWrite], [MarshalEncode],
// [Unmarshal], [UnmarshalRead], and [UnmarshalDecode] with specific features.
// Each function takes in a variadic list of options, where properties
// set in later options override the value of previously set properties.
//
// The Options type is identical to [encoding/json.Options] and
// [encoding/json/jsontext.Options]. Options from the other packages can
// be used interchangeably with functionality in this package.
//
// Options represent either a singular option or a set of options.
// It can be functionally thought of as a Go map of option properties
// (even though the underlying implementation avoids Go maps for performance).
//
// The constructors (e.g., [Deterministic]) return a singular option value:
//
// opt := Deterministic(true)
//
// which is analogous to creating a single entry map:
//
// opt := Options{"Deterministic": true}
//
// [JoinOptions] composes multiple options values together:
//
// out := JoinOptions(opts...)
//
// which is analogous to making a new map and copying the options over:
//
// out := make(Options)
// for _, m := range opts {
// for k, v := range m {
// out[k] = v
// }
// }
//
// [GetOption] looks up the value of an options parameter:
//
// v, ok := GetOption(opts, Deterministic)
//
// which is analogous to a Go map lookup:
//
// v, ok := Options["Deterministic"]
//
// There is a single Options type, which is used with both marshal and unmarshal.
// Some options affect both operations, while others only affect one operation:
//
// - [StringifyNumbers] affects marshaling and unmarshaling
// - [Deterministic] affects marshaling only
// - [FormatNilSliceAsNull] affects marshaling only
// - [FormatNilMapAsNull] affects marshaling only
// - [OmitZeroStructFields] affects marshaling only
// - [MatchCaseInsensitiveNames] affects marshaling and unmarshaling
// - [RejectUnknownMembers] affects unmarshaling only
// - [WithMarshalers] affects marshaling only
// - [WithUnmarshalers] affects unmarshaling only
//
// Options that do not affect a particular operation are ignored.
type Options = json.Options
// JoinOptions coalesces the provided list of options into a single Options.
// Properties set in later options override the value of previously set properties.
func JoinOptions(srcs ...Options) Options {
return json.JoinOptions(srcs...)
}
// GetOption returns the value stored in opts with the provided setter,
// reporting whether the value is present.
// If not present, the returned value is the zero value for type T.
//
// Example usage:
//
// v, ok := json.GetOption(opts, json.Deterministic)
//
// Options are most commonly introspected to alter the JSON representation of
// [MarshalerTo.MarshalJSONTo] and [UnmarshalerFrom.UnmarshalJSONFrom] methods, and
// [MarshalToFunc] and [UnmarshalFromFunc] functions.
// In such cases, the presence bit should generally be ignored.
func GetOption[T any](opts Options, setter func(T) Options) (T, bool) {
return json.GetOption[T](opts, setter)
}
// DefaultOptionsV2 is the full set of all options that define v2 semantics.
// It is equivalent to the set of options in [encoding/json.DefaultOptionsV1]
// all being set to false. All other options are not present.
func DefaultOptionsV2() Options {
return json.DefaultOptionsV2()
}
// StringifyNumbers specifies that types that would normally be
// encoded as a JSON number to instead be encoded as a JSON string
// containing the equivalent JSON number value.
// When unmarshaling, the value is parsed from a JSON string
// containing the JSON number without any surrounding whitespace.
//
// When the `string` tag option is specified on a Go struct field,
// this option is applied for the top-level JSON value for that field.
// Unless StringifyNumbers was applied globally, the option does not
// recursively apply to nested JSON numbers within a JSON object or array.
// A Go type with custom marshal/unmarshal that represents a JSON number
// should respect the StringifyNumbers option and if specified
// serialize as a JSON number within a JSON string.
//
// According to RFC 8259, section 6, a JSON implementation may choose to
// limit the representation of a JSON number to an IEEE 754 binary64 value.
// This may cause decoders to lose precision for int64 and uint64 types.
// Quoting JSON numbers as a JSON string preserves the exact precision.
//
// This affects either marshaling or unmarshaling.
func StringifyNumbers(v bool) Options {
return json.StringifyNumbers(v)
}
// Deterministic specifies that the same input value will be serialized
// as the exact same output bytes. Different processes of
// the same program will serialize equal values to the same bytes,
// but different versions of the same program are not guaranteed
// to produce the exact same sequence of bytes.
//
// This only affects marshaling and is ignored when unmarshaling.
func Deterministic(v bool) Options {
return json.Deterministic(v)
}
// FormatNilSliceAsNull specifies that a nil Go slice should marshal as a
// JSON null instead of the default representation as an empty JSON array
// (or an empty JSON string in the case of ~[]byte).
//
// This only affects marshaling and is ignored when unmarshaling.
func FormatNilSliceAsNull(v bool) Options {
return json.FormatNilSliceAsNull(v)
}
// FormatNilMapAsNull specifies that a nil Go map should marshal as a
// JSON null instead of the default representation as an empty JSON object.
//
// This only affects marshaling and is ignored when unmarshaling.
func FormatNilMapAsNull(v bool) Options {
return json.FormatNilMapAsNull(v)
}
// OmitZeroStructFields specifies that a Go struct should marshal in such a way
// that all struct fields that are zero are omitted from the marshaled output
// if the value is zero as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This is semantically equivalent to specifying the `omitzero` tag option
// on every field in a Go struct.
//
// This only affects marshaling and is ignored when unmarshaling.
func OmitZeroStructFields(v bool) Options {
return json.OmitZeroStructFields(v)
}
// MatchCaseInsensitiveNames specifies that JSON object members are matched
// against Go struct fields using a case-insensitive match of the name.
// If a name matches multiple fields, it chooses the field with an exact
// match of the name, otherwise it reports an error.
// Go struct fields explicitly marked with `case:strict` or `case:ignore`
// always use case-sensitive (or case-insensitive) name matching,
// regardless of the value of this option.
//
// This affects either marshaling or unmarshaling.
//
// By matching names in a case-insensitive manner, it also affects the detection
// of duplicate names (assuming [jsontext.AllowDuplicateNames] is false) since
// variations of the same name may match the same Go struct field.
// For example, when unmarshaling, the names "foo" and "Foo" may both
// match the same Go struct field and therefore be considered a duplicate name.
// When marshaling, normally it is impossible for any two Go struct fields to
// serialize in a way where they unmarshal into the same Go struct field
// since they all have unique exact names.
// However, with the use of an embedded fallback, it is possible for the
// embedded fallback to contain a name that also matches the name for
// a Go struct field, resulting in a duplicate name error.
func MatchCaseInsensitiveNames(v bool) Options {
return json.MatchCaseInsensitiveNames(v)
}
// RejectUnknownMembers specifies that unknown members should be rejected
// when unmarshaling a JSON object.
//
// This only affects unmarshaling and is ignored when marshaling.
func RejectUnknownMembers(v bool) Options {
return json.RejectUnknownMembers(v)
}
// WithMarshalers specifies a list of type-specific marshalers to use,
// which can be used to override the default marshal behavior for values
// of particular types.
//
// This only affects marshaling and is ignored when unmarshaling.
func WithMarshalers(v *Marshalers) Options {
return json.WithMarshalers(v)
}
// WithUnmarshalers specifies a list of type-specific unmarshalers to use,
// which can be used to override the default unmarshal behavior for values
// of particular types.
//
// This only affects unmarshaling and is ignored when marshaling.
func WithUnmarshalers(v *Unmarshalers) Options {
return json.WithUnmarshalers(v)
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

View File

@@ -0,0 +1,569 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"bytes"
"encoding"
"errors"
"io"
"reflect"
"sync"
"time"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/jsontext"
)
// Reference encoding and time packages to assist pkgsite
// in being able to hotlink references to those packages.
var (
_ encoding.TextMarshaler
_ encoding.TextAppender
_ encoding.TextUnmarshaler
_ time.Time
_ time.Duration
)
var (
// Once a JSON object has begun processing without duplicate name verification,
// it does not track the history of names that have been seen so far.
// Reject changing the setting for the current JSON object namespace,
// otherwise we would be operating with inconsistent state.
// Note that you can change the setting before processing the start
// of a different JSON object.
//
// TODO: We could loosen this restriction in certain conditions.
// If we are already checking for duplicate names,
// we can momentarily disable it for the next JSON object member name.
// However, if we are already NOT checking for duplicate names,
// we cannot momentarily enable it for the next JSON object member name
// since we already lack prior history of JSON object names.
errChangingDuplicateNames = errors.New("cannot change duplicate name checks after a JSON object has already begun processing")
// The presence of invalid UTF-8 has an interesting intersection
// with checking for duplicate names. Due to the semantic of mangling
// invalid UTF-8 as the Unicode replacement character,
// two string keys in a Go map (both with invalid UTF-8)
// may encode as the same JSON string. Thus, we forbid changing of
// invalid UTF-8 checks in the current JSON object namespace.
errChangingInvalidUTF8 = errors.New("cannot change UTF-8 checks after a JSON object has already begun processing")
// TODO(https://go.dev/issue/79559): Changing whitespace currently
// leads to strange effects and will need more careful adjustment.
// For now, we just report an error.
errChangingWhitespace = errors.New("cannot change whitespace formatting within a MarshalEncode call")
)
// export exposes internal functionality of the "jsontext" package.
var export = jsontext.Internal.Export(&internal.AllowInternalUse)
// Marshal serializes a Go value as a []byte according to the provided
// marshal and encode options (while ignoring unmarshal or decode options).
// It does not terminate the output with a newline.
//
// Type-specific marshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when encoding
// a value of type T (by taking its address) or a non-nil value of *T.
// Marshal ensures that a value is always addressable
// (by copying the value if necessary) so that
// these functions and methods can be consistently called. For performance,
// it is recommended that Marshal be passed a non-nil pointer to the value.
//
// The input value is encoded as JSON according to the following rules:
//
// - If any type-specific functions in a [WithMarshalers] option match
// the value type, then those functions are called to encode the value.
// If all applicable functions return [errors.ErrUnsupported],
// then the value is encoded according to subsequent rules.
//
// - If the value type implements [MarshalerTo],
// then the MarshalJSONTo method is called to encode the value.
// If the method returns [errors.ErrUnsupported],
// then the input is encoded according to subsequent rules.
//
// - If the value type implements [Marshaler],
// then the MarshalJSON method is called to encode the value.
//
// - If the value type implements [encoding.TextAppender],
// then the AppendText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - If the value type implements [encoding.TextMarshaler],
// then the MarshalText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - Otherwise, the value is encoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation as follows:
//
// - A Go boolean is encoded as a JSON boolean (e.g., true or false).
//
// - A Go string is encoded as a JSON string.
//
// - A Go []byte or [N]byte is encoded as a JSON string containing
// a binary value using Base 64 Encoding per RFC 4648, section 4.
//
// - A Go integer is encoded as a JSON number without fractions or exponents.
// If [StringifyNumbers] is specified or encoding a JSON object name,
// then the JSON number is encoded within a JSON string.
//
// - A Go float is encoded as a JSON number.
// If [StringifyNumbers] is specified or encoding a JSON object name,
// then the JSON number is encoded within a JSON string.
// Encoding a NaN or ±Inf results in a [SemanticError].
//
// - A Go map is encoded as a JSON object, where each Go map key and value
// is recursively encoded as a name and value pair in the JSON object.
// The Go map key must encode as a JSON string, otherwise this results
// in a [SemanticError]. The Go map is traversed in a non-deterministic order.
// For deterministic encoding, consider using the [Deterministic] option.
// By default, a nil map is encoded as an empty JSON object,
// unless the [FormatNilMapAsNull] option is specified.
//
// - A Go struct is encoded as a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is encoded as a JSON array, where each Go slice element
// is recursively JSON-encoded as the elements of the JSON array.
// By default, a nil slice is encoded as an empty JSON array,
// unless the [FormatNilSliceAsNull] option is specified.
//
// - A Go array is encoded as a JSON array, where each Go array element
// is recursively JSON-encoded as the elements of the JSON array.
// The JSON array length is always identical to the Go array length.
//
// - A Go pointer is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
//
// - A Go interface is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
//
// - A Go [time.Time] is encoded as a JSON string containing the timestamp
// formatted in RFC 3339 with nanosecond precision.
//
// - A Go [time.Duration] currently has no default representation and
// results in a [SemanticError], unless the [encoding/json.FormatDurationAsNano]
// option is specified, in which case it is encoded as a JSON number
// without fractions or exponents, representing the duration in nanoseconds.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a [SemanticError].
//
// JSON cannot represent cyclic data structures and Marshal does not handle them.
// Passing cyclic structures will result in an error.
func Marshal(in any, opts ...Options) (out []byte, err error) {
opts = mayAppendSupportFormatTag(opts)
enc := export.GetBufferedEncoder(opts...)
defer export.PutBufferedEncoder(enc)
xe := export.Encoder(enc)
xe.Flags.Set(jsonflags.OmitTopLevelNewline | 1)
err = marshalEncode(enc, in, &xe.Struct)
if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return nil, internal.TransformMarshalError(in, err)
}
return bytes.Clone(xe.Buf), err
}
// MarshalWrite serializes a Go value into an [io.Writer] according to the provided
// marshal and encode options (while ignoring unmarshal or decode options).
// It does not terminate the output with a newline.
// See [Marshal] for details about the conversion of a Go value into JSON.
func MarshalWrite(out io.Writer, in any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
enc := export.GetStreamingEncoder(out, opts...)
defer export.PutStreamingEncoder(enc)
xe := export.Encoder(enc)
xe.Flags.Set(jsonflags.OmitTopLevelNewline | 1)
err = marshalEncode(enc, in, &xe.Struct)
if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.TransformMarshalError(in, err)
}
return err
}
// MarshalEncode serializes a Go value into an [jsontext.Encoder] according to
// the provided marshal or encode options (while ignoring unmarshal or decode options).
// The options provided take precedence over options already applied on
// the [jsontext.Encoder] and only apply for the duration of the marshal call.
//
// See [Marshal] for details about the conversion of a Go value into JSON.
func MarshalEncode(out *jsontext.Encoder, in any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
xe := export.Encoder(out)
if len(opts) > 0 {
optsOriginal := xe.Struct
defer func() { xe.Struct = optsOriginal }()
xe.Struct.Join(opts...)
if xe.Tokens.Last.NeedObjectName() {
if optsOriginal.Flags.Get(jsonflags.AllowDuplicateNames) != xe.Struct.Flags.Get(jsonflags.AllowDuplicateNames) {
return newMarshalErrorBefore(out, reflect.TypeOf(in), errChangingDuplicateNames)
}
if optsOriginal.Flags.Get(jsonflags.AllowInvalidUTF8) != xe.Struct.Flags.Get(jsonflags.AllowInvalidUTF8) {
return newMarshalErrorBefore(out, reflect.TypeOf(in), errChangingInvalidUTF8)
}
}
if xe.Struct.Flags.Has(jsonflags.AnyWhitespace) {
if xe.Struct.Flags.Get(jsonflags.Multiline) {
xe.Struct.InitializeMultiline()
}
if jsonopts.ChangedWhitespace(optsOriginal, xe.Struct) {
return newMarshalErrorBefore(out, reflect.TypeOf(in), errChangingWhitespace)
}
}
}
err = marshalEncode(out, in, &xe.Struct)
if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.TransformMarshalError(in, err)
}
return err
}
func marshalEncode(out *jsontext.Encoder, in any, mo *jsonopts.Struct) (err error) {
v := reflect.ValueOf(in)
if !v.IsValid() || (v.Kind() == reflect.Pointer && v.IsNil()) {
return out.WriteToken(jsontext.Null)
}
// Shallow copy non-pointer values to obtain an addressable value.
// It is beneficial to performance to always pass pointers to avoid this.
forceAddr := v.Kind() != reflect.Pointer
if forceAddr {
v2 := reflect.New(v.Type())
v2.Elem().Set(v)
v = v2
}
va := addressableValue{v.Elem(), forceAddr} // dereferenced pointer is always addressable
t := va.Type()
// Lookup and call the marshal function for this type.
marshal := lookupArshaler(t).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, t)
}
if err := marshal(out, va, mo); err != nil {
if !mo.Flags.Get(jsonflags.AllowDuplicateNames) {
export.Encoder(out).Tokens.InvalidateDisabledNamespaces()
}
return err
}
return nil
}
// Unmarshal decodes a []byte input into a Go value according to the provided
// unmarshal and decode options (while ignoring marshal or encode options).
// The input must be a single JSON value with optional whitespace interspersed.
// The output must be a non-nil pointer.
//
// Type-specific unmarshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when decoding
// a value of type T (by taking its address) or a non-nil value of *T.
// Unmarshal ensures that a value is always addressable
// (by copying the value if necessary) so that
// these functions and methods can be consistently called.
// If a value must be shallow copied to call a pointer-receiver
// [Unmarshaler], [UnmarshalerFrom], or [encoding.TextUnmarshaler] method,
// then any mutations performed by the method are shallow copied back
// into the destination value.
//
// The input is decoded into the output according to the following rules:
//
// - If any type-specific functions in a [WithUnmarshalers] option match
// the value type, then those functions are called to decode the JSON
// value. If all applicable functions return [errors.ErrUnsupported],
// then the input is decoded according to subsequent rules.
//
// - If the value type implements [UnmarshalerFrom],
// then the UnmarshalJSONFrom method is called to decode the JSON value.
// If the method returns [errors.ErrUnsupported],
// then the input is decoded according to subsequent rules.
//
// - If the value type implements [Unmarshaler],
// then the UnmarshalJSON method is called to decode the JSON value.
//
// - If the value type implements [encoding.TextUnmarshaler],
// then the input is decoded as a JSON string and
// the UnmarshalText method is called with the decoded string value.
// This fails with a [SemanticError] if the input is not a JSON string.
//
// - Otherwise, the JSON value is decoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// A JSON null may be decoded into every supported Go value where
// it is equivalent to storing the zero value of the Go value.
// If the input JSON kind is not handled by the current Go value type,
// then this fails with a [SemanticError]. Unless otherwise specified,
// the decoded value replaces any pre-existing value.
//
// The representation of each type is as follows:
//
// - A Go boolean is decoded from a JSON boolean (e.g., true or false).
//
// - A Go string is decoded from a JSON string.
//
// - A Go []byte or [N]byte is decoded from a JSON string containing
// a binary value using Base 64 Encoding per RFC 4648, section 4.
// When decoding into a non-nil []byte, the slice length is reset to zero
// and the decoded input is appended to it.
// When decoding into a [N]byte, the input must decode to exactly N bytes,
// otherwise it fails with a [SemanticError].
//
// - A Go integer is decoded from a JSON number.
// It must be decoded from a JSON string containing a JSON number
// if [StringifyNumbers] is specified or decoding a JSON object name.
// It fails with a [SemanticError] if the JSON number
// has a fractional or exponent component.
// It also fails if it overflows the representation of the Go integer type.
//
// - A Go float is decoded from a JSON number.
// It must be decoded from a JSON string containing a JSON number
// if [StringifyNumbers] is specified or decoding a JSON object name.
// It fails if it overflows the representation of the Go float type.
// Since JSON lacks a native representation for a NaN or ±Inf,
// such values cannot be the result of decoding.
//
// - A Go map is decoded from a JSON object,
// where each JSON object name and value pair is recursively decoded
// as the Go map key and value. Maps are not cleared.
// If the Go map is nil, then a new map is allocated to decode into.
// If the decoded key matches an existing Go map entry, the entry value
// is reused by decoding the JSON object value into it.
//
// - A Go struct is decoded from a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is decoded from a JSON array, where each JSON element
// is recursively decoded and appended to the Go slice.
// Before appending into a Go slice, a new slice is allocated if it is nil,
// otherwise the slice length is reset to zero.
//
// - A Go array is decoded from a JSON array, where each JSON array element
// is recursively decoded as each corresponding Go array element.
// Each Go array element is zeroed before decoding into it.
// It fails with a [SemanticError] if the JSON array does not contain
// the exact same number of elements as the Go array.
//
// - A Go pointer is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil pointer.
// Otherwise, it allocates a new underlying value if the pointer is nil,
// and recursively JSON decodes into the underlying value.
//
// - A Go interface is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil interface value.
// Otherwise, a nil interface value of an empty interface type is initialized
// with a zero Go bool, string, float64, map[string]any, or []any if the
// input is a JSON boolean, string, number, object, or array, respectively.
// If the interface value is still nil, then this fails with a [SemanticError]
// since decoding could not determine an appropriate Go type to decode into.
// For example, unmarshaling into a nil io.Reader fails since
// there is no concrete type to populate the interface value with.
// Otherwise an underlying value exists and it recursively decodes
// the JSON input into it.
//
// - A Go [time.Time] is decoded from a JSON string containing the time
// formatted in RFC 3339 with nanosecond precision.
//
// - A Go [time.Duration] currently has no default representation and
// results in a [SemanticError], unless the [encoding/json.FormatDurationAsNano]
// option is specified, in which case it is decoded as a JSON number
// without fractions or exponents, representing the duration in nanoseconds.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a [SemanticError].
//
// In general, unmarshaling follows merge semantics (similar to RFC 7396)
// where the decoded Go value replaces the destination value
// for any JSON kind other than an object.
// For JSON objects, the input object is merged into the destination value
// where matching object members recursively apply merge semantics.
func Unmarshal(in []byte, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
dec := export.GetBufferedDecoder(in, opts...)
defer export.PutBufferedDecoder(dec)
xd := export.Decoder(dec)
err = unmarshalDecode(dec, out, &xd.Struct, true)
if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.TransformUnmarshalError(out, err)
}
return err
}
// UnmarshalRead deserializes a Go value from an [io.Reader] according to the
// provided unmarshal and decode options (while ignoring marshal or encode options).
// The input must be a single JSON value with optional whitespace interspersed.
// It consumes the entirety of [io.Reader] until [io.EOF] is encountered,
// without reporting an error for EOF. The output must be a non-nil pointer.
// See [Unmarshal] for details about the conversion of JSON into a Go value.
func UnmarshalRead(in io.Reader, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
dec := export.GetStreamingDecoder(in, opts...)
defer export.PutStreamingDecoder(dec)
xd := export.Decoder(dec)
err = unmarshalDecode(dec, out, &xd.Struct, true)
if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.TransformUnmarshalError(out, err)
}
return err
}
// UnmarshalDecode deserializes a Go value from a [jsontext.Decoder] according to
// the provided unmarshal or decode options (while ignoring marshal or encode options).
// The options provided take precedence over options already applied on
// the [jsontext.Decoder] and only apply for the duration of the unmarshal call.
//
// The input may be a stream of zero or more JSON values,
// where this only unmarshals the next JSON value in the stream.
// If there are no more top-level JSON values, it reports [io.EOF].
// The output must be a non-nil pointer.
// See [Unmarshal] for details about the conversion of JSON into a Go value.
func UnmarshalDecode(in *jsontext.Decoder, out any, opts ...Options) (err error) {
opts = mayAppendSupportFormatTag(opts)
xd := export.Decoder(in)
if len(opts) > 0 {
optsOriginal := xd.Struct
defer func() { xd.Struct = optsOriginal }()
xd.Struct.Join(opts...)
if xd.Tokens.Last.NeedObjectName() {
if optsOriginal.Flags.Get(jsonflags.AllowDuplicateNames) != xd.Struct.Flags.Get(jsonflags.AllowDuplicateNames) {
return newUnmarshalErrorBefore(in, reflect.TypeOf(out), errChangingDuplicateNames)
}
if optsOriginal.Flags.Get(jsonflags.AllowInvalidUTF8) != xd.Struct.Flags.Get(jsonflags.AllowInvalidUTF8) {
return newUnmarshalErrorBefore(in, reflect.TypeOf(out), errChangingInvalidUTF8)
}
}
}
err = unmarshalDecode(in, out, &xd.Struct, false)
if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.TransformUnmarshalError(out, err)
}
return err
}
func unmarshalDecode(in *jsontext.Decoder, out any, uo *jsonopts.Struct, last bool) (err error) {
v := reflect.ValueOf(out)
if v.Kind() != reflect.Pointer || v.IsNil() {
return &SemanticError{action: "unmarshal", GoType: reflect.TypeOf(out), Err: internal.ErrNonNilReference}
}
va := addressableValue{v.Elem(), false} // dereferenced pointer is always addressable
t := va.Type()
// In legacy semantics, the entirety of the next JSON value
// was validated before attempting to unmarshal it.
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
if err := export.Decoder(in).CheckNextValue(last); err != nil {
if err == io.EOF && last {
offset := in.InputOffset() + int64(len(in.UnreadBuffer()))
return &jsontext.SyntacticError{ByteOffset: offset, Err: io.ErrUnexpectedEOF}
}
return err
}
}
// Lookup and call the unmarshal function for this type.
unmarshal := lookupArshaler(t).unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, t)
}
if err := unmarshal(in, va, uo); err != nil {
if !uo.Flags.Get(jsonflags.AllowDuplicateNames) {
export.Decoder(in).Tokens.InvalidateDisabledNamespaces()
}
if err == io.EOF && last {
offset := in.InputOffset() + int64(len(in.UnreadBuffer()))
return &jsontext.SyntacticError{ByteOffset: offset, Err: io.ErrUnexpectedEOF}
}
return err
}
if last {
return export.Decoder(in).CheckEOF()
}
return nil
}
// addressableValue is a reflect.Value that is guaranteed to be addressable
// such that calling the Addr and Set methods do not panic.
//
// There is no compile magic that enforces this property,
// but rather the need to construct this type makes it easier to examine each
// construction site to ensure that this property is upheld.
type addressableValue struct {
reflect.Value
// forcedAddr reports whether this value is addressable
// only through the use of [newAddressableValue].
// This is only used for [jsonflags.CallMethodsWithLegacySemantics].
forcedAddr bool
}
// newAddressableValue constructs a new addressable value of type t.
func newAddressableValue(t reflect.Type) addressableValue {
return addressableValue{reflect.New(t).Elem(), true}
}
// TODO: Remove *jsonopts.Struct argument from [marshaler] and [unmarshaler].
// This can be directly accessed on the encoder or decoder.
// All marshal and unmarshal behavior is implemented using these signatures.
// The *jsonopts.Struct argument is guaranteed to be identical to or at least
// a strict super-set of the options in Encoder.Struct or Decoder.Struct.
// It is identical for Marshal, Unmarshal, MarshalWrite, and UnmarshalRead.
// It is a super-set for MarshalEncode and UnmarshalDecode.
type (
marshaler = func(*jsontext.Encoder, addressableValue, *jsonopts.Struct) error
unmarshaler = func(*jsontext.Decoder, addressableValue, *jsonopts.Struct) error
)
type arshaler struct {
marshal marshaler
unmarshal unmarshaler
nonDefault bool
}
var lookupArshalerCache sync.Map // map[reflect.Type]*arshaler
func lookupArshaler(t reflect.Type) *arshaler {
if v, ok := lookupArshalerCache.Load(t); ok {
return v.(*arshaler)
}
fncs := makeDefaultArshaler(t)
fncs = makeMethodArshaler(fncs, t)
fncs = makeTimeArshaler(fncs, t)
// Use the last stored so that duplicate arshalers can be garbage collected.
v, _ := lookupArshalerCache.LoadOrStore(t, fncs)
return v.(*arshaler)
}
var stringsPools = &sync.Pool{New: func() any { return new(stringSlice) }}
type stringSlice []string
// getStrings returns a non-nil pointer to a slice with length n.
func getStrings(n int) *stringSlice {
s := stringsPools.Get().(*stringSlice)
if cap(*s) < n {
*s = make([]string, n)
}
*s = (*s)[:n]
return s
}
func putStrings(s *stringSlice) {
if cap(*s) > 1<<10 {
*s = nil // avoid pinning arbitrarily large amounts of memory
}
clear(*s) // avoid pinning a reference to each string
stringsPools.Put(s)
}

View File

@@ -0,0 +1,289 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"cmp"
"errors"
"math"
"reflect"
"slices"
"strconv"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
"github.com/go-json-experiment/json/jsontext"
)
// This file contains an optimized marshal and unmarshal implementation
// for the any type. This type is often used when the Go program has
// no knowledge of the JSON schema. This is a common enough occurrence
// to justify the complexity of adding logic for this.
// marshalValueAny marshals a Go any as a JSON value.
// This assumes that there are no special formatting directives
// for any possible nested value.
func marshalValueAny(enc *jsontext.Encoder, val any, mo *jsonopts.Struct) error {
switch val := val.(type) {
case nil:
return enc.WriteToken(jsontext.Null)
case bool:
return enc.WriteToken(jsontext.Bool(val))
case string:
return enc.WriteToken(jsontext.String(val))
case float64:
if math.IsNaN(val) || math.IsInf(val, 0) {
break // use default logic below
}
return enc.WriteToken(jsontext.Float(val))
case map[string]any:
return marshalObjectAny(enc, val, mo)
case []any:
return marshalArrayAny(enc, val, mo)
}
v := newAddressableValue(reflect.TypeOf(val))
v.Set(reflect.ValueOf(val))
marshal := lookupArshaler(v.Type()).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, v.Type())
}
return marshal(enc, v, mo)
}
// unmarshalValueAny unmarshals a JSON value as a Go any.
// This assumes that there are no special formatting directives
// for any possible nested value.
// Duplicate names must be rejected since this does not implement merging.
func unmarshalValueAny(dec *jsontext.Decoder, uo *jsonopts.Struct) (any, error) {
switch k := dec.PeekKind(); k {
case '{':
return unmarshalObjectAny(dec, uo)
case '[':
return unmarshalArrayAny(dec, uo)
default:
xd := export.Decoder(dec)
var flags jsonwire.ValueFlags
val, err := xd.ReadValue(&flags)
if err != nil {
return nil, err
}
switch val.Kind() {
case 'n':
return nil, nil
case 'f':
return false, nil
case 't':
return true, nil
case '"':
val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
if xd.StringCache == nil {
xd.StringCache = new(stringCache)
}
return makeString(xd.StringCache, val), nil
case '0':
if uo.Flags.Get(jsonflags.UnmarshalAnyWithRawNumber) {
return internal.RawNumberOf(val), nil
}
fv, err := strconv.ParseFloat(string(val), 64)
if err != nil {
return fv, newUnmarshalErrorAfterWithValue(dec, float64Type, errors.Unwrap(err))
}
return fv, nil
default:
panic("BUG: invalid kind: " + k.String())
}
}
}
// marshalObjectAny marshals a Go map[string]any as a JSON object
// (or as a JSON null if nil and [jsonflags.FormatNilMapAsNull]).
func marshalObjectAny(enc *jsontext.Encoder, obj map[string]any, mo *jsonopts.Struct) error {
// Check for cycles.
xe := export.Encoder(enc)
if xe.Tokens.Depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(obj)
if err := visitPointer(&xe.SeenPointers, v); err != nil {
return newMarshalErrorBefore(enc, mapStringAnyType, err)
}
defer leavePointer(&xe.SeenPointers, v)
}
// Handle empty maps.
if len(obj) == 0 {
if mo.Flags.Get(jsonflags.FormatNilMapAsNull) && obj == nil {
return enc.WriteToken(jsontext.Null)
}
// Optimize for marshaling an empty map without any preceding whitespace.
if !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() {
xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '{'), "{}"...)
xe.Tokens.Last.Increment()
if xe.NeedFlush() {
return xe.Flush()
}
return nil
}
}
if err := enc.WriteToken(jsontext.BeginObject); err != nil {
return err
}
// A Go map guarantees that each entry has a unique key.
// The only possibility of duplicates is due to invalid UTF-8.
if !mo.Flags.Get(jsonflags.AllowInvalidUTF8) {
xe.Tokens.Last.DisableNamespace()
}
if !mo.Flags.Get(jsonflags.Deterministic) || len(obj) <= 1 {
for name, val := range obj {
if err := enc.WriteToken(jsontext.String(name)); err != nil {
return err
}
if err := marshalValueAny(enc, val, mo); err != nil {
return err
}
}
} else {
names := getStrings(len(obj))
var i int
for name := range obj {
(*names)[i] = name
i++
}
slices.Sort(*names)
for _, name := range *names {
if err := enc.WriteToken(jsontext.String(name)); err != nil {
return err
}
if err := marshalValueAny(enc, obj[name], mo); err != nil {
return err
}
}
putStrings(names)
}
if err := enc.WriteToken(jsontext.EndObject); err != nil {
return err
}
return nil
}
// unmarshalObjectAny unmarshals a JSON object as a Go map[string]any.
// It panics if not decoding a JSON object.
func unmarshalObjectAny(dec *jsontext.Decoder, uo *jsonopts.Struct) (map[string]any, error) {
switch tok, err := dec.ReadToken(); {
case err != nil:
return nil, err
case tok.Kind() != '{':
panic("BUG: invalid kind: " + tok.Kind().String())
}
obj := make(map[string]any)
// A Go map guarantees that each entry has a unique key.
// The only possibility of duplicates is due to invalid UTF-8.
if !uo.Flags.Get(jsonflags.AllowInvalidUTF8) {
export.Decoder(dec).Tokens.Last.DisableNamespace()
}
var errUnmarshal error
for dec.PeekKind() != '}' {
tok, err := dec.ReadToken()
if err != nil {
return obj, err
}
name := tok.String()
// Manually check for duplicate names.
if _, ok := obj[name]; ok {
// TODO: Unread the object name.
name := export.Decoder(dec).PreviousTokenOrValue()
err := newDuplicateNameError(dec.StackPointer(), nil, dec.InputOffset()-len64(name))
return obj, err
}
val, err := unmarshalValueAny(dec, uo)
obj[name] = val
if err != nil {
if isFatalError(err, uo.Flags) {
return obj, err
}
errUnmarshal = cmp.Or(err, errUnmarshal)
}
}
if _, err := dec.ReadToken(); err != nil {
return obj, err
}
return obj, errUnmarshal
}
// marshalArrayAny marshals a Go []any as a JSON array
// (or as a JSON null if nil and [jsonflags.FormatNilSliceAsNull]).
func marshalArrayAny(enc *jsontext.Encoder, arr []any, mo *jsonopts.Struct) error {
// Check for cycles.
xe := export.Encoder(enc)
if xe.Tokens.Depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(arr)
if err := visitPointer(&xe.SeenPointers, v); err != nil {
return newMarshalErrorBefore(enc, sliceAnyType, err)
}
defer leavePointer(&xe.SeenPointers, v)
}
// Handle empty slices.
if len(arr) == 0 {
if mo.Flags.Get(jsonflags.FormatNilSliceAsNull) && arr == nil {
return enc.WriteToken(jsontext.Null)
}
// Optimize for marshaling an empty slice without any preceding whitespace.
if !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() {
xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '['), "[]"...)
xe.Tokens.Last.Increment()
if xe.NeedFlush() {
return xe.Flush()
}
return nil
}
}
if err := enc.WriteToken(jsontext.BeginArray); err != nil {
return err
}
for _, val := range arr {
if err := marshalValueAny(enc, val, mo); err != nil {
return err
}
}
if err := enc.WriteToken(jsontext.EndArray); err != nil {
return err
}
return nil
}
// unmarshalArrayAny unmarshals a JSON array as a Go []any.
// It panics if not decoding a JSON array.
func unmarshalArrayAny(dec *jsontext.Decoder, uo *jsonopts.Struct) ([]any, error) {
switch tok, err := dec.ReadToken(); {
case err != nil:
return nil, err
case tok.Kind() != '[':
panic("BUG: invalid kind: " + tok.Kind().String())
}
arr := []any{}
var errUnmarshal error
for dec.PeekKind() != ']' {
val, err := unmarshalValueAny(dec, uo)
arr = append(arr, val)
if err != nil {
if isFatalError(err, uo.Flags) {
return arr, err
}
errUnmarshal = cmp.Or(errUnmarshal, err)
}
}
if _, err := dec.ReadToken(); err != nil {
return arr, err
}
return arr, errUnmarshal
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,230 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"bytes"
"errors"
"io"
"reflect"
"slices"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
"github.com/go-json-experiment/json/jsontext"
)
// This package supports "embedding" a Go struct field, where the contents
// of the serialized field (which must be a JSON object) are treated as if
// they are part of the parent Go struct (which represents a JSON object).
//
// Generally, embedded fields are of a Go struct type, where the fields of the
// nested struct are virtually hoisted up to the parent struct using rules
// similar to how Go embedding works (but operating within the JSON namespace).
//
// However, embedded fields may also be of a Go map type with a string key or
// a jsontext.Value. Such embedded fields are called "fallback" fields since they
// represent any arbitrary JSON object member. Explicitly named fields take
// precedence over the embedded fallback. Only one embedded fallback is allowed.
var errRawEmbedNotObject = errors.New("embedded raw value must be a JSON object")
var jsontextValueType = reflect.TypeFor[jsontext.Value]()
// marshalEmbeddedFallbackAll marshals all the members in an embedded fallback.
func marshalEmbeddedFallbackAll(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct, f *structField, insertUnquotedName func([]byte) bool) error {
v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable
if len(f.index) > 0 {
v = v.fieldByIndex(f.index, false)
if !v.IsValid() {
return nil // implies a nil embedded field
}
}
v = v.indirect(false)
if !v.IsValid() {
return nil
}
if v.Type() == jsontextValueType {
b, _ := reflect.TypeAssert[jsontext.Value](v.Value)
if len(b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
return nil
}
dec := export.GetBufferedDecoder(b)
defer export.PutBufferedDecoder(dec)
xd := export.Decoder(dec)
xd.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1)
tok, err := dec.ReadToken()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return newMarshalErrorBefore(enc, v.Type(), err)
}
if tok.Kind() != '{' {
return newMarshalErrorBefore(enc, v.Type(), errRawEmbedNotObject)
}
for dec.PeekKind() != '}' {
// Parse the JSON object name.
var flags jsonwire.ValueFlags
val, err := xd.ReadValue(&flags)
if err != nil {
return newMarshalErrorBefore(enc, v.Type(), err)
}
if insertUnquotedName != nil {
name := jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
if !insertUnquotedName(name) {
return newDuplicateNameError(enc.StackPointer().Parent(), val, enc.OutputOffset())
}
}
if err := enc.WriteValue(val); err != nil {
return err
}
// Parse the JSON object value.
val, err = xd.ReadValue(&flags)
if err != nil {
return newMarshalErrorBefore(enc, v.Type(), err)
}
if err := enc.WriteValue(val); err != nil {
return err
}
}
if _, err := dec.ReadToken(); err != nil {
return newMarshalErrorBefore(enc, v.Type(), err)
}
if err := xd.CheckEOF(); err != nil {
return newMarshalErrorBefore(enc, v.Type(), err)
}
return nil
} else {
m := v // must be a map[~string]V
n := m.Len()
if n == 0 {
return nil
}
mk := newAddressableValue(m.Type().Key())
mv := newAddressableValue(m.Type().Elem())
marshalKey := func(mk addressableValue) error {
b, err := jsonwire.AppendQuote(enc.AvailableBuffer(), []byte(mk.String()), &mo.Flags)
if err != nil {
return newMarshalErrorBefore(enc, m.Type().Key(), err)
}
if insertUnquotedName != nil {
isVerbatim := bytes.IndexByte(b, '\\') < 0
name := jsonwire.UnquoteMayCopy(b, isVerbatim)
if !insertUnquotedName(name) {
return newDuplicateNameError(enc.StackPointer().Parent(), b, enc.OutputOffset())
}
}
return enc.WriteValue(b)
}
marshalVal := f.fncs.marshal
if mo.Marshalers != nil {
marshalVal, _ = mo.Marshalers.(*Marshalers).lookup(marshalVal, mv.Type())
}
if !mo.Flags.Get(jsonflags.Deterministic) || n <= 1 {
for iter := m.MapRange(); iter.Next(); {
mk.SetIterKey(iter)
if err := marshalKey(mk); err != nil {
return err
}
mv.Set(iter.Value())
if err := marshalVal(enc, mv, mo); err != nil {
return err
}
}
} else {
names := getStrings(n)
for i, iter := 0, m.Value.MapRange(); i < n && iter.Next(); i++ {
mk.SetIterKey(iter)
(*names)[i] = mk.String()
}
slices.Sort(*names)
for _, name := range *names {
mk.SetString(name)
if err := marshalKey(mk); err != nil {
return err
}
// TODO(https://go.dev/issue/57061): Use mv.SetMapIndexOf.
mv.Set(m.MapIndex(mk.Value))
if err := marshalVal(enc, mv, mo); err != nil {
return err
}
}
putStrings(names)
}
return nil
}
}
// unmarshalEmbeddedFallbackNext unmarshals only the next member in an embedded fallback.
func unmarshalEmbeddedFallbackNext(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct, f *structField, quotedName, unquotedName []byte) error {
v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable
if len(f.index) > 0 {
v = v.fieldByIndex(f.index, true)
}
v = v.indirect(true)
if v.Type() == jsontextValueType {
b, _ := reflect.TypeAssert[*jsontext.Value](v.Addr())
if len(*b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
*b = append(*b, '{')
} else {
*b = jsonwire.TrimSuffixWhitespace(*b)
if jsonwire.HasSuffixByte(*b, '}') {
// TODO: When merging into an object for the first time,
// should we verify that it is valid?
*b = jsonwire.TrimSuffixByte(*b, '}')
*b = jsonwire.TrimSuffixWhitespace(*b)
if !jsonwire.HasSuffixByte(*b, ',') && !jsonwire.HasSuffixByte(*b, '{') {
*b = append(*b, ',')
}
} else {
return newUnmarshalErrorAfterWithSkipping(dec, v.Type(), errRawEmbedNotObject)
}
}
*b = append(*b, quotedName...)
*b = append(*b, ':')
val, err := dec.ReadValue()
if err != nil {
return err
}
*b = append(*b, val...)
*b = append(*b, '}')
return nil
} else {
name := string(unquotedName) // TODO: Intern this?
m := v // must be a map[~string]V
if m.IsNil() {
m.Set(reflect.MakeMap(m.Type()))
}
mk := reflect.ValueOf(name)
if mkt := m.Type().Key(); mkt != stringType {
mk = mk.Convert(mkt)
}
mv := newAddressableValue(m.Type().Elem()) // TODO: Cache across calls?
if v2 := m.MapIndex(mk); v2.IsValid() {
mv.Set(v2)
}
unmarshal := f.fncs.unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, mv.Type())
}
err := unmarshal(dec, mv, uo)
m.SetMapIndex(mk, mv.Value)
if err != nil {
return err
}
return nil
}
}

View File

@@ -0,0 +1,428 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"errors"
"fmt"
"io"
"reflect"
"sync"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/jsontext"
)
var errUnsupportedMutation = errors.New("unsupported calls must not read or write any tokens")
var errNonSingularValue = errors.New("must read or write exactly one value")
// Marshalers is a list of functions that may override the marshal behavior
// of specific types. Populate [WithMarshalers] to use it with
// [Marshal], [MarshalWrite], or [MarshalEncode].
// A nil *Marshalers is equivalent to an empty list.
// There are no exported fields or methods on Marshalers.
type Marshalers = typedMarshalers
// JoinMarshalers constructs a flattened list of marshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns [errors.ErrUnsupported],
// then the next applicable function is called,
// otherwise the default marshaling behavior is used.
//
// For example:
//
// m1 := JoinMarshalers(f1, f2)
// m2 := JoinMarshalers(f0, m1, f3) // equivalent to m3
// m3 := JoinMarshalers(f0, f1, f2, f3) // equivalent to m2
func JoinMarshalers(ms ...*Marshalers) *Marshalers {
return newMarshalers(ms...)
}
// Unmarshalers is a list of functions that may override the unmarshal behavior
// of specific types. Populate [WithUnmarshalers] to use it with
// [Unmarshal], [UnmarshalRead], or [UnmarshalDecode].
// A nil *Unmarshalers is equivalent to an empty list.
// There are no exported fields or methods on Unmarshalers.
type Unmarshalers = typedUnmarshalers
// JoinUnmarshalers constructs a flattened list of unmarshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns [errors.ErrUnsupported],
// then the next applicable function is called,
// otherwise the default unmarshaling behavior is used.
//
// For example:
//
// u1 := JoinUnmarshalers(f1, f2)
// u2 := JoinUnmarshalers(f0, u1, f3) // equivalent to u3
// u3 := JoinUnmarshalers(f0, f1, f2, f3) // equivalent to u2
func JoinUnmarshalers(us ...*Unmarshalers) *Unmarshalers {
return newUnmarshalers(us...)
}
type typedMarshalers = typedArshalers[jsontext.Encoder]
type typedUnmarshalers = typedArshalers[jsontext.Decoder]
type typedArshalers[Coder any] struct {
nonComparable
fncVals []typedArshaler[Coder]
fncCache sync.Map // map[reflect.Type]arshaler
// fromAny reports whether any of Go types used to represent arbitrary JSON
// (i.e., any, bool, string, float64, map[string]any, or []any) matches
// any of the provided type-specific arshalers.
//
// This bit of information is needed in arshal_default.go to determine
// whether to use the specialized logic in arshal_any.go to handle
// the any interface type. The logic in arshal_any.go does not support
// type-specific arshal functions, so we must avoid using that logic
// if this is true.
fromAny bool
}
type typedMarshaler = typedArshaler[jsontext.Encoder]
type typedUnmarshaler = typedArshaler[jsontext.Decoder]
type typedArshaler[Coder any] struct {
typ reflect.Type
fnc func(*Coder, addressableValue, *jsonopts.Struct) error
maySkip bool
}
func newMarshalers(ms ...*Marshalers) *Marshalers { return newTypedArshalers(ms...) }
func newUnmarshalers(us ...*Unmarshalers) *Unmarshalers { return newTypedArshalers(us...) }
func newTypedArshalers[Coder any](as ...*typedArshalers[Coder]) *typedArshalers[Coder] {
var a typedArshalers[Coder]
for _, a2 := range as {
if a2 != nil {
a.fncVals = append(a.fncVals, a2.fncVals...)
a.fromAny = a.fromAny || a2.fromAny
}
}
if len(a.fncVals) == 0 {
return nil
}
return &a
}
func (a *typedArshalers[Coder]) lookup(fnc func(*Coder, addressableValue, *jsonopts.Struct) error, t reflect.Type) (func(*Coder, addressableValue, *jsonopts.Struct) error, bool) {
if a == nil {
return fnc, false
}
if v, ok := a.fncCache.Load(t); ok {
if v == nil {
return fnc, false
}
return v.(func(*Coder, addressableValue, *jsonopts.Struct) error), true
}
// Collect a list of arshalers that can be called for this type.
// This list may be longer than 1 since some arshalers can be skipped.
var fncs []func(*Coder, addressableValue, *jsonopts.Struct) error
for _, fncVal := range a.fncVals {
if !castableTo(t, fncVal.typ) {
continue
}
fncs = append(fncs, fncVal.fnc)
if !fncVal.maySkip {
break // subsequent arshalers will never be called
}
}
if len(fncs) == 0 {
a.fncCache.Store(t, nil) // nil to indicate that no funcs found
return fnc, false
}
// Construct an arshaler that may call every applicable arshaler.
fncDefault := fnc
fnc = func(c *Coder, v addressableValue, o *jsonopts.Struct) error {
for _, fnc := range fncs {
if err := fnc(c, v, o); !errors.Is(err, errors.ErrUnsupported) {
return err // may be nil or non-nil
}
}
return fncDefault(c, v, o)
}
// Use the first stored so duplicate work can be garbage collected.
v, _ := a.fncCache.LoadOrStore(t, fnc)
return v.(func(*Coder, addressableValue, *jsonopts.Struct) error), true
}
// MarshalFunc constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value.
// The value of T must not be retained outside the function call.
// It is recommended that fn return a []byte buffer that is safe
// for the caller to retain and potentially mutate.
// It may not return [errors.ErrUnsupported].
func MarshalFunc[T any](fn func(T) ([]byte, error)) *Marshalers {
t := reflect.TypeFor[T]()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
v, _ := reflect.TypeAssert[T](va.castTo(t))
val, err := fn(v)
if err != nil {
err = wrapErrUnsupported(err, "marshal function of type func(T) ([]byte, error)")
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalFunc") // unlike unmarshal, always wrapped
}
err = newMarshalErrorBefore(enc, t, err)
return collapseSemanticErrors(err)
}
if err := enc.WriteValue(val); err != nil {
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalFunc") // unlike unmarshal, always wrapped
}
if isSyntacticError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
},
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// MarshalToFunc constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value by calling write methods
// on the provided encoder. It may return [errors.ErrUnsupported] such that marshaling can
// move on to the next marshal function. However, no mutable method calls may
// be called on the encoder if [errors.ErrUnsupported] is returned.
// The pointer to [jsontext.Encoder] and the value of T
// must not be retained outside the function call.
func MarshalToFunc[T any](fn func(*jsontext.Encoder, T) error) *Marshalers {
t := reflect.TypeFor[T]()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
xe := export.Encoder(enc)
prevDepth, prevLength := xe.Tokens.DepthLength()
xe.Flags.Set(jsonflags.WithinArshalCall | 1)
v, _ := reflect.TypeAssert[T](va.castTo(t))
err := fn(enc, v)
xe.Flags.Set(jsonflags.WithinArshalCall | 0)
currDepth, currLength := xe.Tokens.DepthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errNonSingularValue
}
if err != nil {
if errors.Is(err, errors.ErrUnsupported) {
if prevDepth == currDepth && prevLength == currLength {
return err // forward [errors.ErrUnsupported]
}
err = errUnsupportedMutation
}
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalToFunc") // unlike unmarshal, always wrapped
}
if !export.IsIOError(err) {
err = newSemanticErrorWithPosition(enc, t, prevDepth, prevLength, err)
}
return err
}
return nil
},
maySkip: true,
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFunc constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value.
// The input []byte must not be mutated.
// The input []byte and value T must not be retained outside the function call.
// It may not return [errors.ErrUnsupported].
func UnmarshalFunc[T any](fn func([]byte, T) error) *Unmarshalers {
t := reflect.TypeFor[T]()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
v, _ := reflect.TypeAssert[T](va.castTo(t))
err = fn(val, v)
if err != nil {
err = wrapErrUnsupported(err, "unmarshal function of type func([]byte, T) error")
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return err // unlike marshal, never wrapped
}
err = newUnmarshalErrorAfter(dec, t, err)
return collapseSemanticErrors(err)
}
return nil
},
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFromFunc constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value by calling read methods
// on the provided decoder. It may return [errors.ErrUnsupported] such that unmarshaling can
// move on to the next unmarshal function. However, no mutable method calls may
// be called on the decoder if [errors.ErrUnsupported] is returned.
// The pointer to [jsontext.Decoder] and the value of T
// must not be retained outside the function call.
func UnmarshalFromFunc[T any](fn func(*jsontext.Decoder, T) error) *Unmarshalers {
t := reflect.TypeFor[T]()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
xd := export.Decoder(dec)
prevDepth, prevLength := xd.Tokens.DepthLength()
if prevDepth == 1 && xd.AtEOF() {
return io.EOF // check EOF early to avoid fn reporting an EOF
}
xd.Flags.Set(jsonflags.WithinArshalCall | 1)
v, _ := reflect.TypeAssert[T](va.castTo(t))
err := fn(dec, v)
xd.Flags.Set(jsonflags.WithinArshalCall | 0)
currDepth, currLength := xd.Tokens.DepthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errNonSingularValue
}
if err != nil {
if errors.Is(err, errors.ErrUnsupported) {
if prevDepth == currDepth && prevLength == currLength {
return err // forward [errors.ErrUnsupported]
}
err = errUnsupportedMutation
}
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
if err2 := xd.SkipUntil(prevDepth, prevLength+1); err2 != nil {
return err2
}
return err // unlike marshal, never wrapped
}
if !isSyntacticError(err) && !export.IsIOError(err) {
err = newSemanticErrorWithPosition(dec, t, prevDepth, prevLength, err)
}
return err
}
return nil
},
maySkip: true,
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// assertCastableTo asserts that "to" is a valid type to be casted to.
// These are the Go types that type-specific arshalers may operate upon.
//
// Let AllTypes be the universal set of all possible Go types.
// This function generally asserts that:
//
// len([from for from in AllTypes if castableTo(from, to)]) > 0
//
// otherwise it panics.
//
// As a special-case if marshal is false, then we forbid any non-pointer or
// non-interface type since it is almost always a bug trying to unmarshal
// into something where the end-user caller did not pass in an addressable value
// since they will not observe the mutations.
func assertCastableTo(to reflect.Type, marshal bool) {
switch to.Kind() {
case reflect.Interface:
return
case reflect.Pointer:
// Only allow unnamed pointers to be consistent with the fact that
// taking the address of a value produces an unnamed pointer type.
if to.Name() == "" {
return
}
default:
// Technically, non-pointer types are permissible for unmarshal.
// However, they are often a bug since the receiver would be immutable.
// Thus, only allow them for marshaling.
if marshal {
return
}
}
if marshal {
panic(fmt.Sprintf("input type %v must be an interface type, an unnamed pointer type, or a non-pointer type", to))
} else {
panic(fmt.Sprintf("input type %v must be an interface type or an unnamed pointer type", to))
}
}
// castableTo checks whether values of type "from" can be casted to type "to".
// Nil pointer or interface "from" values are never considered castable.
//
// This function must be kept in sync with addressableValue.castTo.
func castableTo(from, to reflect.Type) bool {
switch to.Kind() {
case reflect.Interface:
// TODO: This breaks when ordinary interfaces can have type sets
// since interfaces now exist where only the value form of a type (T)
// implements the interface, but not the pointer variant (*T).
// See https://go.dev/issue/45346.
return reflect.PointerTo(from).Implements(to)
case reflect.Pointer:
// Common case for unmarshaling.
// From must be a concrete or interface type.
return reflect.PointerTo(from) == to
default:
// Common case for marshaling.
// From must be a concrete type.
return from == to
}
}
// castTo casts va to the specified type.
// If the type is an interface, then the underlying type will always
// be a non-nil pointer to a concrete type.
//
// Requirement: castableTo(va.Type(), to) must hold.
func (va addressableValue) castTo(to reflect.Type) reflect.Value {
switch to.Kind() {
case reflect.Interface:
return va.Addr().Convert(to)
case reflect.Pointer:
return va.Addr()
default:
return va.Value
}
}
// castableToFromAny reports whether "to" can be casted to from any
// of the dynamic types used to represent arbitrary JSON.
func castableToFromAny(to reflect.Type) bool {
for _, from := range []reflect.Type{anyType, boolType, stringType, float64Type, mapStringAnyType, sliceAnyType} {
if castableTo(from, to) {
return true
}
}
return false
}

View File

@@ -0,0 +1,378 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"encoding"
"errors"
"io"
"reflect"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
"github.com/go-json-experiment/json/jsontext"
)
var errNonStringValue = errors.New("JSON value must be string type")
// Interfaces for custom serialization.
var (
jsonMarshalerType = reflect.TypeFor[Marshaler]()
jsonMarshalerToType = reflect.TypeFor[MarshalerTo]()
jsonUnmarshalerType = reflect.TypeFor[Unmarshaler]()
jsonUnmarshalerFromType = reflect.TypeFor[UnmarshalerFrom]()
textAppenderType = reflect.TypeFor[encoding.TextAppender]()
textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]()
textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]()
allMarshalerTypes = []reflect.Type{jsonMarshalerToType, jsonMarshalerType, textAppenderType, textMarshalerType}
allUnmarshalerTypes = []reflect.Type{jsonUnmarshalerFromType, jsonUnmarshalerType, textUnmarshalerType}
allMethodTypes = append(allMarshalerTypes, allUnmarshalerTypes...)
)
// Marshaler is implemented by types that can marshal themselves.
// It is recommended that types implement [MarshalerTo] unless the implementation
// is trying to avoid a hard dependency on the "jsontext" package.
//
// It is recommended that implementations return a buffer that is safe
// for the caller to retain and potentially mutate.
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError].
type Marshaler interface {
MarshalJSON() ([]byte, error)
}
// MarshalerTo is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerTo instead of [Marshaler]
// since this is both more performant and flexible.
// If a type implements both Marshaler and MarshalerTo,
// then MarshalerTo takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default marshal options.
//
// The implementation must write only one JSON value to the Encoder.
// Alternatively, it may return [errors.ErrUnsupported] without mutating
// the Encoder. The "json" package calling the method will
// use the next available JSON representation for the receiver type.
// Implementations must not retain the pointer to [jsontext.Encoder].
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError],
// unless it is an IO error.
//
// The MarshalJSONTo method should not be directly called as it may
// return sentinel errors that need special handling.
// Users should instead call [MarshalEncode], which handles such cases.
type MarshalerTo interface {
MarshalJSONTo(*jsontext.Encoder) error
}
// Unmarshaler is implemented by types that can unmarshal themselves.
// It is recommended that types implement [UnmarshalerFrom] unless the implementation
// is trying to avoid a hard dependency on the "jsontext" package.
//
// The input can be assumed to be a valid encoding of a JSON value
// if called from unmarshal functionality in this package.
// UnmarshalJSON must copy the JSON data if it is retained after returning.
// It is recommended that UnmarshalJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain or mutate the input []byte.
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError].
type Unmarshaler interface {
UnmarshalJSON([]byte) error
}
// UnmarshalerFrom is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerFrom instead of [Unmarshaler]
// since this is both more performant and flexible.
// If a type implements both Unmarshaler and UnmarshalerFrom,
// then UnmarshalerFrom takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default unmarshal options.
//
// The implementation must read only one JSON value from the Decoder.
// It is recommended that UnmarshalJSONFrom implement merge semantics when
// unmarshaling into a pre-populated value.
// Alternatively, it may return [errors.ErrUnsupported] without mutating
// the Decoder. The "json" package calling the method will
// use the next available JSON representation for the receiver type.
// Implementations must not retain the pointer to [jsontext.Decoder].
//
// If the returned error is a [SemanticError], then unpopulated fields
// of the error may be populated by [json] with additional context.
// Errors of other types are wrapped within a [SemanticError],
// unless it is a [jsontext.SyntacticError] or an IO error.
//
// The UnmarshalJSONFrom method should not be directly called as it may
// return sentinel errors that need special handling.
// Users should instead call [UnmarshalDecode], which handles such cases.
type UnmarshalerFrom interface {
UnmarshalJSONFrom(*jsontext.Decoder) error
}
func makeMethodArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Avoid injecting method arshaler on the pointer or interface version
// to avoid ever calling the method on a nil pointer or interface receiver.
// Let it be injected on the value receiver (which is always addressable).
if t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface {
return fncs
}
if needAddr, ok := implements(t, textMarshalerType); ok {
fncs.nonDefault = true
prevMarshal := fncs.marshal
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
(needAddr && va.forcedAddr) {
return prevMarshal(enc, va, mo)
}
marshaler, _ := reflect.TypeAssert[encoding.TextMarshaler](va.Addr())
if err := export.Encoder(enc).AppendRaw('"', false, func(b []byte) ([]byte, error) {
b2, err := marshaler.MarshalText()
return append(b, b2...), err
}); err != nil {
err = wrapErrUnsupported(err, "MarshalText method")
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalText") // unlike unmarshal, always wrapped
}
if !isSemanticError(err) && !export.IsIOError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
}
}
if needAddr, ok := implements(t, textAppenderType); ok {
fncs.nonDefault = true
prevMarshal := fncs.marshal
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) (err error) {
if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
(needAddr && va.forcedAddr) {
return prevMarshal(enc, va, mo)
}
appender, _ := reflect.TypeAssert[encoding.TextAppender](va.Addr())
if err := export.Encoder(enc).AppendRaw('"', false, appender.AppendText); err != nil {
err = wrapErrUnsupported(err, "AppendText method")
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "AppendText") // unlike unmarshal, always wrapped
}
if !isSemanticError(err) && !export.IsIOError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
}
}
if needAddr, ok := implements(t, jsonMarshalerType); ok {
fncs.nonDefault = true
prevMarshal := fncs.marshal
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
((needAddr && va.forcedAddr) || export.Encoder(enc).Tokens.Last.NeedObjectName()) {
return prevMarshal(enc, va, mo)
}
marshaler, _ := reflect.TypeAssert[Marshaler](va.Addr())
val, err := marshaler.MarshalJSON()
if err != nil {
err = wrapErrUnsupported(err, "MarshalJSON method")
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped
}
err = newMarshalErrorBefore(enc, t, err)
return collapseSemanticErrors(err)
}
if err := enc.WriteValue(val); err != nil {
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped
}
if isSyntacticError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
}
}
if needAddr, ok := implements(t, jsonMarshalerToType); ok {
fncs.nonDefault = true
prevMarshal := fncs.marshal
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
((needAddr && va.forcedAddr) || export.Encoder(enc).Tokens.Last.NeedObjectName()) {
return prevMarshal(enc, va, mo)
}
xe := export.Encoder(enc)
prevDepth, prevLength := xe.Tokens.DepthLength()
xe.Flags.Set(jsonflags.WithinArshalCall | 1)
marshaler, _ := reflect.TypeAssert[MarshalerTo](va.Addr())
err := marshaler.MarshalJSONTo(enc)
xe.Flags.Set(jsonflags.WithinArshalCall | 0)
currDepth, currLength := xe.Tokens.DepthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errNonSingularValue
}
if err != nil {
if errors.Is(err, errors.ErrUnsupported) {
if prevDepth == currDepth && prevLength == currLength {
return prevMarshal(enc, va, mo)
}
err = errUnsupportedMutation
}
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSONTo") // unlike unmarshal, always wrapped
}
if !export.IsIOError(err) {
err = newSemanticErrorWithPosition(enc, t, prevDepth, prevLength, err)
}
return err
}
return nil
}
}
if _, ok := implements(t, textUnmarshalerType); ok {
fncs.nonDefault = true
fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
xd := export.Decoder(dec)
var flags jsonwire.ValueFlags
val, err := xd.ReadValue(&flags)
if err != nil {
return err // must be a syntactic or I/O error
}
if val.Kind() == 'n' {
if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) {
va.SetZero()
}
return nil
}
if val.Kind() != '"' {
return newUnmarshalErrorAfter(dec, t, errNonStringValue)
}
s := jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
unmarshaler, _ := reflect.TypeAssert[encoding.TextUnmarshaler](va.Addr())
if err := unmarshaler.UnmarshalText(s); err != nil {
err = wrapErrUnsupported(err, "UnmarshalText method")
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return err // unlike marshal, never wrapped
}
if !isSemanticError(err) && !isSyntacticError(err) && !export.IsIOError(err) {
err = newUnmarshalErrorAfter(dec, t, err)
}
return err
}
return nil
}
}
if _, ok := implements(t, jsonUnmarshalerType); ok {
fncs.nonDefault = true
prevUnmarshal := fncs.unmarshal
fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
if uo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
export.Decoder(dec).Tokens.Last.NeedObjectName() {
return prevUnmarshal(dec, va, uo)
}
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
unmarshaler, _ := reflect.TypeAssert[Unmarshaler](va.Addr())
if err := unmarshaler.UnmarshalJSON(val); err != nil {
err = wrapErrUnsupported(err, "UnmarshalJSON method")
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return err // unlike marshal, never wrapped
}
err = newUnmarshalErrorAfter(dec, t, err)
return collapseSemanticErrors(err)
}
return nil
}
}
if _, ok := implements(t, jsonUnmarshalerFromType); ok {
fncs.nonDefault = true
prevUnmarshal := fncs.unmarshal
fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
if uo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) &&
export.Decoder(dec).Tokens.Last.NeedObjectName() {
return prevUnmarshal(dec, va, uo)
}
xd := export.Decoder(dec)
prevDepth, prevLength := xd.Tokens.DepthLength()
if prevDepth == 1 && xd.AtEOF() {
return io.EOF // check EOF early to avoid fn reporting an EOF
}
xd.Flags.Set(jsonflags.WithinArshalCall | 1)
unmarshaler, _ := reflect.TypeAssert[UnmarshalerFrom](va.Addr())
err := unmarshaler.UnmarshalJSONFrom(dec)
xd.Flags.Set(jsonflags.WithinArshalCall | 0)
currDepth, currLength := xd.Tokens.DepthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errNonSingularValue
}
if err != nil {
if errors.Is(err, errors.ErrUnsupported) {
if prevDepth == currDepth && prevLength == currLength {
return prevUnmarshal(dec, va, uo)
}
err = errUnsupportedMutation
}
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
if err2 := xd.SkipUntil(prevDepth, prevLength+1); err2 != nil {
return err2
}
return err // unlike marshal, never wrapped
}
if !isSyntacticError(err) && !export.IsIOError(err) {
err = newSemanticErrorWithPosition(dec, t, prevDepth, prevLength, err)
}
return err
}
return nil
}
}
return fncs
}
// implementsAny is like t.Implements(ifaceType) for a list of interfaces,
// but checks whether either t or reflect.PointerTo(t) implements the interface.
func implementsAny(t reflect.Type, ifaceTypes ...reflect.Type) bool {
for _, ifaceType := range ifaceTypes {
if _, ok := implements(t, ifaceType); ok {
return true
}
}
return false
}
// implements is like t.Implements(ifaceType) but checks whether
// either t or reflect.PointerTo(t) implements the interface.
// It also reports whether the value needs to be addressed
// in order to satisfy the interface.
func implements(t, ifaceType reflect.Type) (needAddr, ok bool) {
switch {
case t.Implements(ifaceType):
return false, true
case reflect.PointerTo(t).Implements(ifaceType):
return true, true
default:
return false, false
}
}

View File

@@ -0,0 +1,795 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"bytes"
"cmp"
"errors"
"fmt"
"math"
"math/bits"
"reflect"
"strconv"
"strings"
"time"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
"github.com/go-json-experiment/json/jsontext"
)
var (
timeDurationType = reflect.TypeFor[time.Duration]()
timeTimeType = reflect.TypeFor[time.Time]()
)
func makeTimeArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Ideally, time types would implement MarshalerTo and UnmarshalerFrom,
// but that would incur a dependency on package json from package time.
// Given how widely used time is, it is more acceptable that we incur a
// dependency on time from json.
//
// Injecting the arshaling functionality like this will not be identical
// to actually declaring methods on the time types since embedding of the
// time types will not be able to forward this functionality.
switch t {
case timeDurationType:
fncs.nonDefault = true
marshalNano := fncs.marshal
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
xe := export.Encoder(enc)
var m durationArshaler
if mo.Flags.Has(jsonflags.FormatTag) {
if !m.initFormat(mo.Format) {
return newInvalidFormatError(enc, t)
}
} else if mo.Flags.Get(jsonflags.FormatDurationAsNano) {
return marshalNano(enc, va, mo)
} else {
// TODO(https://go.dev/issue/71631): Decide on default duration representation.
var workaround string
if mo.Flags.Get(jsonflags.FormatTagSupported) {
workaround = "; specify an explicit format"
}
return newMarshalErrorBefore(enc, t, errors.New("no default representation"+workaround))
}
if mo.Flags.Get(jsonflags.StringTag) && !m.isNumeric() && !mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return newMarshalErrorBefore(enc, t, errInvalidStringTag)
}
m.td, _ = reflect.TypeAssert[time.Duration](va.Value)
k := stringOrNumberKind(!m.isNumeric() || xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringTag))
if err := xe.AppendRaw(k, true, m.appendMarshal); err != nil {
if !isSyntacticError(err) && !export.IsIOError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
}
unmarshalNano := fncs.unmarshal
fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
xd := export.Decoder(dec)
var u durationArshaler
if uo.Flags.Has(jsonflags.FormatTag) {
if !u.initFormat(uo.Format) {
return newInvalidFormatError(dec, t)
}
} else if uo.Flags.Get(jsonflags.FormatDurationAsNano) {
return unmarshalNano(dec, va, uo)
} else {
// TODO(https://go.dev/issue/71631): Decide on default duration representation.
var workaround string
if uo.Flags.Get(jsonflags.FormatTagSupported) {
workaround = "; specify an explicit format"
}
return newUnmarshalErrorBeforeWithSkipping(dec, t, errors.New("no default representation"+workaround))
}
if uo.Flags.Get(jsonflags.StringTag) && !u.isNumeric() && !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return newUnmarshalErrorBeforeWithSkipping(dec, t, errInvalidStringTag)
}
stringify := !u.isNumeric() || xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringTag)
var flags jsonwire.ValueFlags
td, _ := reflect.TypeAssert[*time.Duration](va.Addr())
val, err := xd.ReadValue(&flags)
if err != nil {
return err
}
switch k := val.Kind(); k {
case 'n':
if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) {
*td = time.Duration(0)
}
return nil
case '"':
if !stringify {
break
}
val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
if err := u.unmarshal(val); err != nil {
return newUnmarshalErrorAfter(dec, t, err)
}
*td = u.td
return nil
case '0':
if stringify {
break
}
if err := u.unmarshal(val); err != nil {
return newUnmarshalErrorAfter(dec, t, err)
}
*td = u.td
return nil
}
return newUnmarshalErrorAfter(dec, t, nil)
}
case timeTimeType:
fncs.nonDefault = true
fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) (err error) {
xe := export.Encoder(enc)
var m timeArshaler
if mo.Flags.Has(jsonflags.FormatTag) {
if !m.initFormat(mo.Format) {
return newInvalidFormatError(enc, t)
}
}
if mo.Flags.Get(jsonflags.StringTag) && !m.isNumeric() && !mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return newMarshalErrorBefore(enc, t, errInvalidStringTag)
}
m.tt, _ = reflect.TypeAssert[time.Time](va.Value)
k := stringOrNumberKind(!m.isNumeric() || xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringTag))
if err := xe.AppendRaw(k, !m.hasCustomFormat(), m.appendMarshal); err != nil {
if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped
}
if !isSyntacticError(err) && !export.IsIOError(err) {
err = newMarshalErrorBefore(enc, t, err)
}
return err
}
return nil
}
fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) (err error) {
xd := export.Decoder(dec)
var u timeArshaler
if uo.Flags.Has(jsonflags.FormatTag) {
if !u.initFormat(uo.Format) {
return newInvalidFormatError(dec, t)
}
} else if uo.Flags.Get(jsonflags.ParseTimeWithLooseRFC3339) {
u.looseRFC3339 = true
}
if uo.Flags.Get(jsonflags.StringTag) && !u.isNumeric() && !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return newUnmarshalErrorBeforeWithSkipping(dec, t, errInvalidStringTag)
}
stringify := !u.isNumeric() || xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringTag)
var flags jsonwire.ValueFlags
tt, _ := reflect.TypeAssert[*time.Time](va.Addr())
val, err := xd.ReadValue(&flags)
if err != nil {
return err
}
switch k := val.Kind(); k {
case 'n':
if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) {
*tt = time.Time{}
}
return nil
case '"':
if !stringify {
break
}
val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
if err := u.unmarshal(val); err != nil {
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return err // unlike marshal, never wrapped
}
return newUnmarshalErrorAfter(dec, t, err)
}
*tt = u.tt
return nil
case '0':
if stringify {
break
}
if err := u.unmarshal(val); err != nil {
if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
return err // unlike marshal, never wrapped
}
return newUnmarshalErrorAfter(dec, t, err)
}
*tt = u.tt
return nil
}
return newUnmarshalErrorAfter(dec, t, nil)
}
}
return fncs
}
type durationArshaler struct {
td time.Duration
// base records the representation where:
// - 0 uses time.Duration.String
// - 1e0, 1e3, 1e6, or 1e9 use a decimal encoding of the duration as
// nanoseconds, microseconds, milliseconds, or seconds.
// - 8601 uses ISO 8601
base uint64
}
func (a *durationArshaler) initFormat(format string) (ok bool) {
switch format {
case "units":
a.base = 0
case "sec":
a.base = 1e9
case "milli":
a.base = 1e6
case "micro":
a.base = 1e3
case "nano":
a.base = 1e0
case "iso8601":
a.base = 8601
default:
return false
}
return true
}
func (a *durationArshaler) isNumeric() bool {
return a.base != 0 && a.base != 8601
}
func (a *durationArshaler) appendMarshal(b []byte) ([]byte, error) {
switch a.base {
case 0:
return append(b, a.td.String()...), nil
case 8601:
return appendDurationISO8601(b, a.td), nil
default:
return appendDurationBase10(b, a.td, a.base), nil
}
}
func (a *durationArshaler) unmarshal(b []byte) (err error) {
switch a.base {
case 0:
a.td, err = time.ParseDuration(string(b))
case 8601:
a.td, err = parseDurationISO8601(b)
default:
a.td, err = parseDurationBase10(b, a.base)
}
return err
}
type timeArshaler struct {
tt time.Time
// base records the representation where:
// - 0 uses RFC 3339 encoding of the timestamp
// - 1e0, 1e3, 1e6, or 1e9 use a decimal encoding of the timestamp as
// seconds, milliseconds, microseconds, or nanoseconds since Unix epoch.
// - math.MaxUint uses time.Time.Format to encode the timestamp
base uint64
format string // time format passed to time.Parse
looseRFC3339 bool
}
func (a *timeArshaler) initFormat(format string) bool {
// We assume that an exported constant in the time package will
// always start with an uppercase ASCII letter.
if len(format) == 0 {
return false
}
a.base = math.MaxUint // implies custom format
if c := format[0]; !('a' <= c && c <= 'z') && !('A' <= c && c <= 'Z') {
a.format = format
return true
}
switch format {
case "ANSIC":
a.format = time.ANSIC
case "UnixDate":
a.format = time.UnixDate
case "RubyDate":
a.format = time.RubyDate
case "RFC822":
a.format = time.RFC822
case "RFC822Z":
a.format = time.RFC822Z
case "RFC850":
a.format = time.RFC850
case "RFC1123":
a.format = time.RFC1123
case "RFC1123Z":
a.format = time.RFC1123Z
case "RFC3339":
a.base = 0
a.format = time.RFC3339
case "RFC3339Nano":
a.base = 0
a.format = time.RFC3339Nano
case "Kitchen":
a.format = time.Kitchen
case "Stamp":
a.format = time.Stamp
case "StampMilli":
a.format = time.StampMilli
case "StampMicro":
a.format = time.StampMicro
case "StampNano":
a.format = time.StampNano
case "DateTime":
a.format = time.DateTime
case "DateOnly":
a.format = time.DateOnly
case "TimeOnly":
a.format = time.TimeOnly
case "unix":
a.base = 1e0
case "unixmilli":
a.base = 1e3
case "unixmicro":
a.base = 1e6
case "unixnano":
a.base = 1e9
default:
// Reject any Go identifier in case new constants are supported.
if strings.TrimFunc(format, isLetterOrDigit) == "" {
return false
}
a.format = format
}
return true
}
func (a *timeArshaler) isNumeric() bool {
return int(a.base) > 0
}
func (a *timeArshaler) hasCustomFormat() bool {
return a.base == math.MaxUint
}
func (a *timeArshaler) appendMarshal(b []byte) ([]byte, error) {
switch a.base {
case 0:
format := cmp.Or(a.format, time.RFC3339Nano)
n0 := len(b)
b = a.tt.AppendFormat(b, format)
// Not all Go timestamps can be represented as valid RFC 3339.
// Explicitly check for these edge cases.
// See https://go.dev/issue/4556 and https://go.dev/issue/54580.
switch b := b[n0:]; {
case b[len("9999")] != '-': // year must be exactly 4 digits wide
return b, errors.New("year outside of range [0,9999]")
case b[len(b)-1] != 'Z':
c := b[len(b)-len("Z07:00")]
if ('0' <= c && c <= '9') || parseDec2(b[len(b)-len("07:00"):]) >= 24 {
return b, errors.New("timezone hour outside of range [0,23]")
}
}
return b, nil
case math.MaxUint:
return a.tt.AppendFormat(b, a.format), nil
default:
return appendTimeUnix(b, a.tt, a.base), nil
}
}
func (a *timeArshaler) unmarshal(b []byte) (err error) {
switch a.base {
case 0:
// Use time.Time.UnmarshalText to avoid possible string allocation.
if err := a.tt.UnmarshalText(b); err != nil {
return err
}
// TODO(https://go.dev/issue/57912):
// RFC 3339 specifies the grammar for a valid timestamp.
// However, the parsing functionality in "time" is too loose and
// incorrectly accepts invalid timestamps as valid.
// Remove these manual checks when "time" checks it for us.
newParseError := func(layout, value, layoutElem, valueElem, message string) error {
return &time.ParseError{Layout: layout, Value: value, LayoutElem: layoutElem, ValueElem: valueElem, Message: message}
}
switch {
case a.looseRFC3339:
return nil
case b[len("2006-01-02T")+1] == ':': // hour must be two digits
return newParseError(time.RFC3339, string(b), "15", string(b[len("2006-01-02T"):][:1]), "")
case b[len("2006-01-02T15:04:05")] == ',': // sub-second separator must be a period
return newParseError(time.RFC3339, string(b), ".", ",", "")
case b[len(b)-1] != 'Z':
switch {
case parseDec2(b[len(b)-len("07:00"):]) >= 24: // timezone hour must be in range
return newParseError(time.RFC3339, string(b), "Z07:00", string(b[len(b)-len("Z07:00"):]), ": timezone hour out of range")
case parseDec2(b[len(b)-len("00"):]) >= 60: // timezone minute must be in range
return newParseError(time.RFC3339, string(b), "Z07:00", string(b[len(b)-len("Z07:00"):]), ": timezone minute out of range")
}
}
return nil
case math.MaxUint:
a.tt, err = time.Parse(a.format, string(b))
return err
default:
a.tt, err = parseTimeUnix(b, a.base)
return err
}
}
// appendDurationBase10 appends d formatted as a decimal fractional number,
// where pow10 is a power-of-10 used to scale down the number.
func appendDurationBase10(b []byte, d time.Duration, pow10 uint64) []byte {
b, n := mayAppendDurationSign(b, d) // append sign
whole, frac := bits.Div64(0, n, uint64(pow10)) // compute whole and frac fields
b = strconv.AppendUint(b, whole, 10) // append whole field
return appendFracBase10(b, frac, pow10) // append frac field
}
// parseDurationBase10 parses d from a decimal fractional number,
// where pow10 is a power-of-10 used to scale up the number.
func parseDurationBase10(b []byte, pow10 uint64) (time.Duration, error) {
suffix, neg := consumeSign(b, false) // consume sign
wholeBytes, fracBytes := bytesCutByte(suffix, '.', true) // consume whole and frac fields
whole, okWhole := jsonwire.ParseUint(wholeBytes) // parse whole field; may overflow
frac, okFrac := parseFracBase10(fracBytes, pow10) // parse frac field
hi, lo := bits.Mul64(whole, uint64(pow10)) // overflow if hi > 0
sum, co := bits.Add64(lo, uint64(frac), 0) // overflow if co > 0
switch d := mayApplyDurationSign(sum, neg); { // overflow if neg != (d < 0)
case (!okWhole && whole != math.MaxUint64) || !okFrac:
return 0, fmt.Errorf("invalid duration %q: %w", b, strconv.ErrSyntax)
case !okWhole || hi > 0 || co > 0 || neg != (d < 0):
return 0, fmt.Errorf("invalid duration %q: %w", b, strconv.ErrRange)
default:
return d, nil
}
}
// appendDurationISO8601 appends an ISO 8601 duration with a restricted grammar,
// where leading and trailing zeroes and zero-value designators are omitted.
// It only uses hour, minute, and second designators since ISO 8601 defines
// those as being "accurate", while year, month, week, and day are "nominal".
func appendDurationISO8601(b []byte, d time.Duration) []byte {
if d == 0 {
return append(b, "PT0S"...)
}
b, n := mayAppendDurationSign(b, d)
b = append(b, "PT"...)
n, nsec := bits.Div64(0, n, 1e9) // compute nsec field
n, sec := bits.Div64(0, n, 60) // compute sec field
hour, min := bits.Div64(0, n, 60) // compute hour and min fields
if hour > 0 {
b = append(strconv.AppendUint(b, hour, 10), 'H')
}
if min > 0 {
b = append(strconv.AppendUint(b, min, 10), 'M')
}
if sec > 0 || nsec > 0 {
b = append(appendFracBase10(strconv.AppendUint(b, sec, 10), nsec, 1e9), 'S')
}
return b
}
// daysPerYear is the exact average number of days in a year according to
// the Gregorian calendar, which has an extra day each year that is
// a multiple of 4, unless it is evenly divisible by 100 but not by 400.
// This does not take into account leap seconds, which are not deterministic.
const daysPerYear = 365.2425
var errInaccurateDateUnits = errors.New("inaccurate year, month, week, or day units")
// parseDurationISO8601 parses a duration according to ISO 8601-1:2019,
// section 5.5.2.2 and 5.5.2.3 with the following restrictions or extensions:
//
// - A leading minus sign is permitted for negative duration according
// to ISO 8601-2:2019, section 4.4.1.9. We do not permit negative values
// for each "time scale component", which is permitted by section 4.4.1.1,
// but rarely supported by parsers.
//
// - A leading plus sign is permitted (and ignored).
// This is not required by ISO 8601, but not forbidden either.
// There is some precedent for this as it is supported by the principle of
// duration arithmetic as specified in ISO 8601-2-2019, section 14.1.
// Of note, the JavaScript grammar for ISO 8601 permits a leading plus sign.
//
// - A fractional value is only permitted for accurate units
// (i.e., hour, minute, and seconds) in the last time component,
// which is permissible by ISO 8601-1:2019, section 5.5.2.3.
//
// - Both periods ('.') and commas (',') are supported as the separator
// between the integer part and fraction part of a number,
// as specified in ISO 8601-1:2019, section 3.2.6.
// While ISO 8601 recommends comma as the default separator,
// most formatters use a period.
//
// - Leading zeros are ignored. This is not required by ISO 8601,
// but also not forbidden by the standard. Many parsers support this.
//
// - Lowercase designators are supported. This is not required by ISO 8601,
// but also not forbidden by the standard. Many parsers support this.
//
// If the nominal units of year, month, week, or day are present,
// this produces a best-effort value and also reports [errInaccurateDateUnits].
//
// The accepted grammar is identical to JavaScript's Duration:
//
// https://tc39.es/proposal-temporal/#prod-Duration
//
// We follow JavaScript's grammar as JSON itself is derived from JavaScript.
// The Temporal.Duration.toJSON method is guaranteed to produce an output
// that can be parsed by this function so long as arithmetic in JavaScript
// does not use a largestUnit value higher than "hours" (which is the default).
// Even if it does, this will do a best-effort parsing with inaccurate units,
// but report [errInaccurateDateUnits].
func parseDurationISO8601(b []byte) (time.Duration, error) {
var invalid, overflow, inaccurate, sawFrac bool
var sumNanos, n, co uint64
// cutBytes is like [bytes.Cut], but uses either c0 or c1 as the separator.
cutBytes := func(b []byte, c0, c1 byte) (prefix, suffix []byte, ok bool) {
for i, c := range b {
if c == c0 || c == c1 {
return b[:i], b[i+1:], true
}
}
return b, nil, false
}
// mayParseUnit attempts to parse another date or time number
// identified by the desHi and desLo unit characters.
// If the part is absent for current unit, it returns b as is.
mayParseUnit := func(b []byte, desHi, desLo byte, unit time.Duration) []byte {
number, suffix, ok := cutBytes(b, desHi, desLo)
if !ok || sawFrac {
return b // designator is not present or already saw fraction, which can only be in the last component
}
// Parse the number.
// A fraction is allowed for the accurate units in the last part.
whole, frac, ok := cutBytes(number, '.', ',')
if ok {
sawFrac = true
invalid = invalid || len(frac) == len("") || unit > time.Hour
if unit == time.Second {
n, ok = parsePaddedBase10(frac, uint64(time.Second))
invalid = invalid || !ok
} else {
f, err := strconv.ParseFloat("0."+string(frac), 64)
invalid = invalid || err != nil || len(bytes.Trim(frac[len("."):], "0123456789")) > 0
n = uint64(math.Round(f * float64(unit))) // never overflows since f is within [0..1]
}
sumNanos, co = bits.Add64(sumNanos, n, 0) // overflow if co > 0
overflow = overflow || co > 0
}
for len(whole) > 1 && whole[0] == '0' {
whole = whole[len("0"):] // trim leading zeros
}
n, ok := jsonwire.ParseUint(whole) // overflow if !ok && MaxUint64
hi, lo := bits.Mul64(n, uint64(unit)) // overflow if hi > 0
sumNanos, co = bits.Add64(sumNanos, lo, 0) // overflow if co > 0
invalid = invalid || (!ok && n != math.MaxUint64)
overflow = overflow || (!ok && n == math.MaxUint64) || hi > 0 || co > 0
inaccurate = inaccurate || unit > time.Hour
return suffix
}
suffix, neg := consumeSign(b, true)
prefix, suffix, okP := cutBytes(suffix, 'P', 'p')
durDate, durTime, okT := cutBytes(suffix, 'T', 't')
invalid = invalid || len(prefix) > 0 || !okP || (okT && len(durTime) == 0) || len(durDate)+len(durTime) == 0
if len(durDate) > 0 { // nominal portion of the duration
durDate = mayParseUnit(durDate, 'Y', 'y', time.Duration(daysPerYear*24*60*60*1e9))
durDate = mayParseUnit(durDate, 'M', 'm', time.Duration(daysPerYear/12*24*60*60*1e9))
durDate = mayParseUnit(durDate, 'W', 'w', time.Duration(7*24*60*60*1e9))
durDate = mayParseUnit(durDate, 'D', 'd', time.Duration(24*60*60*1e9))
invalid = invalid || len(durDate) > 0 // unknown elements
}
if len(durTime) > 0 { // accurate portion of the duration
durTime = mayParseUnit(durTime, 'H', 'h', time.Duration(60*60*1e9))
durTime = mayParseUnit(durTime, 'M', 'm', time.Duration(60*1e9))
durTime = mayParseUnit(durTime, 'S', 's', time.Duration(1e9))
invalid = invalid || len(durTime) > 0 // unknown elements
}
d := mayApplyDurationSign(sumNanos, neg)
overflow = overflow || (neg != (d < 0) && d != 0) // overflows signed duration
switch {
case invalid:
return 0, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, strconv.ErrSyntax)
case overflow:
return 0, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, strconv.ErrRange)
case inaccurate:
return d, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, errInaccurateDateUnits)
default:
return d, nil
}
}
// mayAppendDurationSign appends a negative sign if n is negative.
func mayAppendDurationSign(b []byte, d time.Duration) ([]byte, uint64) {
if d < 0 {
b = append(b, '-')
d *= -1
}
return b, uint64(d)
}
// mayApplyDurationSign inverts n if neg is specified.
func mayApplyDurationSign(n uint64, neg bool) time.Duration {
if neg {
return -1 * time.Duration(n)
} else {
return +1 * time.Duration(n)
}
}
// appendTimeUnix appends t formatted as a decimal fractional number,
// where pow10 is a power-of-10 used to scale up the number.
func appendTimeUnix(b []byte, t time.Time, pow10 uint64) []byte {
sec, nsec := t.Unix(), int64(t.Nanosecond())
if sec < 0 {
b = append(b, '-')
sec, nsec = negateSecNano(sec, nsec)
}
switch {
case pow10 == 1e0: // fast case where units is in seconds
b = strconv.AppendUint(b, uint64(sec), 10)
return appendFracBase10(b, uint64(nsec), 1e9)
case uint64(sec) < 1e9: // intermediate case where units is not seconds, but no overflow
b = strconv.AppendUint(b, uint64(sec)*uint64(pow10)+uint64(uint64(nsec)/(1e9/pow10)), 10)
return appendFracBase10(b, (uint64(nsec)*pow10)%1e9, 1e9)
default: // slow case where units is not seconds and overflow would occur
b = strconv.AppendUint(b, uint64(sec), 10)
b = appendPaddedBase10(b, uint64(nsec)/(1e9/pow10), pow10)
return appendFracBase10(b, (uint64(nsec)*pow10)%1e9, 1e9)
}
}
// parseTimeUnix parses t formatted as a decimal fractional number,
// where pow10 is a power-of-10 used to scale down the number.
func parseTimeUnix(b []byte, pow10 uint64) (time.Time, error) {
suffix, neg := consumeSign(b, false) // consume sign
wholeBytes, fracBytes := bytesCutByte(suffix, '.', true) // consume whole and frac fields
whole, okWhole := jsonwire.ParseUint(wholeBytes) // parse whole field; may overflow
frac, okFrac := parseFracBase10(fracBytes, 1e9/pow10) // parse frac field
var sec, nsec int64
switch {
case pow10 == 1e0: // fast case where units is in seconds
sec = int64(whole) // check overflow later after negation
nsec = int64(frac) // cannot overflow
case okWhole: // intermediate case where units is not seconds, but no overflow
sec = int64(whole / pow10) // check overflow later after negation
nsec = int64((whole%pow10)*(1e9/pow10) + frac) // cannot overflow
case !okWhole && whole == math.MaxUint64: // slow case where units is not seconds and overflow occurred
width := int(math.Log10(float64(pow10))) // compute len(strconv.Itoa(pow10-1))
whole, okWhole = jsonwire.ParseUint(wholeBytes[:len(wholeBytes)-width]) // parse the upper whole field
mid, _ := parsePaddedBase10(wholeBytes[len(wholeBytes)-width:], pow10) // parse the lower whole field
sec = int64(whole) // check overflow later after negation
nsec = int64(mid*(1e9/pow10) + frac) // cannot overflow
}
if neg {
sec, nsec = negateSecNano(sec, nsec)
}
switch t := time.Unix(sec, nsec).UTC(); {
case (!okWhole && whole != math.MaxUint64) || !okFrac:
return time.Time{}, fmt.Errorf("invalid time %q: %w", b, strconv.ErrSyntax)
case !okWhole || neg != (t.Unix() < 0):
return time.Time{}, fmt.Errorf("invalid time %q: %w", b, strconv.ErrRange)
default:
return t, nil
}
}
// negateSecNano negates a Unix timestamp, where nsec must be within [0, 1e9).
func negateSecNano(sec, nsec int64) (int64, int64) {
sec = ^sec // twos-complement negation (i.e., -1*sec + 1)
nsec = -nsec + 1e9 // negate nsec and add 1e9 (which is the extra +1 from sec negation)
sec += int64(nsec / 1e9) // handle possible overflow of nsec if it started as zero
nsec %= 1e9 // ensure nsec stays within [0, 1e9)
return sec, nsec
}
// appendFracBase10 appends the fraction of n/max10,
// where max10 is a power-of-10 that is larger than n.
func appendFracBase10(b []byte, n, max10 uint64) []byte {
if n == 0 {
return b
}
return bytes.TrimRight(appendPaddedBase10(append(b, '.'), n, max10), "0")
}
// parseFracBase10 parses the fraction of n/max10,
// where max10 is a power-of-10 that is larger than n.
func parseFracBase10(b []byte, max10 uint64) (n uint64, ok bool) {
switch {
case len(b) == 0:
return 0, true
case len(b) < len(".0") || b[0] != '.':
return 0, false
}
return parsePaddedBase10(b[len("."):], max10)
}
// appendPaddedBase10 appends a zero-padded encoding of n,
// where max10 is a power-of-10 that is larger than n.
func appendPaddedBase10(b []byte, n, max10 uint64) []byte {
if n < max10/10 {
// Formatting of n is shorter than log10(max10),
// so add max10/10 to ensure the length is equal to log10(max10).
i := len(b)
b = strconv.AppendUint(b, n+max10/10, 10)
b[i]-- // subtract the addition of max10/10
return b
}
return strconv.AppendUint(b, n, 10)
}
// parsePaddedBase10 parses b as the zero-padded encoding of n,
// where max10 is a power-of-10 that is larger than n.
// Truncated suffix is treated as implicit zeros.
// Extended suffix is ignored, but verified to contain only digits.
func parsePaddedBase10(b []byte, max10 uint64) (n uint64, ok bool) {
pow10 := uint64(1)
for pow10 < max10 {
n *= 10
if len(b) > 0 {
if b[0] < '0' || '9' < b[0] {
return n, false
}
n += uint64(b[0] - '0')
b = b[1:]
}
pow10 *= 10
}
if len(b) > 0 && len(bytes.TrimRight(b, "0123456789")) > 0 {
return n, false // trailing characters are not digits
}
return n, true
}
// consumeSign consumes an optional leading negative or positive sign.
func consumeSign(b []byte, allowPlus bool) ([]byte, bool) {
if len(b) > 0 {
if b[0] == '-' {
return b[len("-"):], true
} else if b[0] == '+' && allowPlus {
return b[len("+"):], false
}
}
return b, false
}
// bytesCutByte is similar to bytes.Cut(b, []byte{c}),
// except c may optionally be included as part of the suffix.
func bytesCutByte(b []byte, c byte, include bool) ([]byte, []byte) {
if i := bytes.IndexByte(b, c); i >= 0 {
if include {
return b[:i], b[i:]
}
return b[:i], b[i+1:]
}
return b, nil
}
// parseDec2 parses b as an unsigned, base-10, 2-digit number.
// The result is undefined if digits are not base-10.
func parseDec2(b []byte) byte {
if len(b) < 2 {
return 0
}
return 10*(b[0]-'0') + (b[1] - '0')
}

View File

@@ -0,0 +1,247 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
// Package json implements semantic processing of JSON as specified in RFC 8259.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
// [Marshal] and [Unmarshal] encode and decode Go values
// to/from JSON text contained within a []byte.
// [MarshalWrite] and [UnmarshalRead] operate on JSON text
// by writing to or reading from an [io.Writer] or [io.Reader].
// [MarshalEncode] and [UnmarshalDecode] operate on JSON text
// by encoding to or decoding from a [jsontext.Encoder] or [jsontext.Decoder].
// [Options] may be passed to each of the marshal or unmarshal functions
// to configure the semantic behavior of marshaling and unmarshaling
// (i.e., alter how JSON data is understood as Go data and vice versa).
// [jsontext.Options] may also be passed to the marshal or unmarshal functions
// to configure the syntactic behavior of encoding or decoding.
//
// The data types of JSON are mapped to/from the data types of Go based on
// the closest logical equivalent between the two type systems. For example,
// a JSON boolean corresponds with a Go bool,
// a JSON string corresponds with a Go string,
// a JSON number corresponds with a Go int, uint or float,
// a JSON array corresponds with a Go slice or array, and
// a JSON object corresponds with a Go struct or map.
// See the documentation on [Marshal] and [Unmarshal] for a comprehensive list
// of how the JSON and Go type systems correspond.
//
// Arbitrary Go types can customize their JSON representation by implementing
// [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom].
// This provides authors of Go types with control over how their types are
// serialized as JSON. Alternatively, users can implement functions that match
// [MarshalFunc], [MarshalToFunc], [UnmarshalFunc], or [UnmarshalFromFunc]
// to specify the JSON representation for arbitrary types.
// This provides callers of JSON functionality with control over
// how any arbitrary type is serialized as JSON.
//
// # JSON Representation of Go structs
//
// A Go struct is naturally represented as a JSON object,
// where each Go struct field corresponds with a JSON object member.
// When marshaling, all Go struct fields are recursively encoded in depth-first
// order as JSON object members except those that are ignored or omitted.
// When unmarshaling, JSON object members are recursively decoded
// into the corresponding Go struct fields.
// Object members that do not match any struct fields,
// also known as “unknown members”, are ignored by default or rejected
// if [RejectUnknownMembers] is specified.
//
// The representation of each struct field can be customized in the
// "json" struct field tag, where the tag is a comma separated list of options.
// As a special case, if the entire tag is `json:"-"`,
// then the field is ignored with regard to its JSON representation.
// Some options also have equivalent behavior controlled by a caller-specified [Options].
// Field-specified options take precedence over caller-specified options.
//
// The first option is the JSON object name override for the Go struct field.
// If the name is not specified, then the Go struct field name
// is used as the JSON object name.
// By default, unmarshaling uses case-sensitive matching to identify
// the Go struct field associated with a JSON object name.
//
// After the name, the following tag options are supported:
//
// - omitzero: When marshaling, the "omitzero" option specifies that
// the struct field should be omitted if the field value is zero
// as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This option has no effect when unmarshaling.
//
// - omitempty: When marshaling, the "omitempty" option specifies that
// the struct field should be omitted if the field value would have been
// encoded as a JSON null, empty string, empty object, or empty array.
// This option has no effect when unmarshaling.
//
// - string: The "string" option specifies that [StringifyNumbers] be set
// when marshaling or unmarshaling a struct field value.
// This causes types that would normally be encoded as a JSON number
// to instead be encoded as a JSON number quoted within a JSON string,
// and to be decoded from a JSON string containing the JSON number
// without any surrounding whitespace.
// The "string" option only applies to the top-level of the Go struct field value.
// Specifically, for the default representation of composite Go data types
// (e.g., array, slice, struct, or map), it will not stringify JSON numbers
// within such types. Applying this option to invalid types causes a runtime error.
// This extra level of encoding is often necessary since many JSON parsers
// cannot precisely represent 64-bit integers.
//
// - case: When unmarshaling, the "case" option specifies how
// JSON object names are matched with the JSON name for Go struct fields.
// The option is a key-value pair specified as "case:value" where
// the value must either be 'ignore' or 'strict'.
// The 'ignore' value specifies that matching is case-insensitive
// where dashes and underscores are also ignored. If multiple fields match,
// then the field with an exact name match is selected, otherwise an error
// is reported due to an ambiguous set of candidate fields to unmarshal into.
// The 'strict' value specifies that matching is case-sensitive.
// This takes precedence over the [MatchCaseInsensitiveNames] option.
//
// - embed: The "embed" option specifies that
// the JSON representable content of this field type is to be promoted
// as if they were specified in the parent struct.
// It is the JSON equivalent of Go struct embedding.
// A Go embedded field is implicitly JSON embedded unless
// an explicit JSON name is specified. The embedded field must be a Go struct
// (that does not implement any JSON methods), [jsontext.Value],
// map[~string]T, or an unnamed pointer to such types. When marshaling,
// embedded fields from a pointer type are omitted if it is nil.
// Embedded fields of type [jsontext.Value] and map[~string]T are called
// “embedded fallbacks” as they can represent all possible
// JSON object members not directly handled by the parent struct.
// Only one embedded fallback field may be specified in a struct,
// while many non-fallback fields may be specified. This option
// must not be specified with any other option (including the JSON name).
//
// The "omitzero" and "omitempty" options are mostly semantically identical.
// The former is defined in terms of the Go type system,
// while the latter in terms of the JSON type system.
// Consequently they behave differently in some circumstances.
// For example, only a nil slice or map is omitted under "omitzero", while
// an empty slice or map is omitted under "omitempty" regardless of nilness.
// The "omitzero" option is useful for types with a well-defined zero value
// (e.g., [net/netip.Addr]) or have an IsZero method (e.g., [time.Time.IsZero]).
//
// Every Go struct corresponds to a list of JSON representable fields
// which is constructed by performing a breadth-first search over
// all struct fields (excluding unexported or ignored fields),
// where the search recursively descends into embedded structs.
// The set of non-embedded fields in a struct must have unique JSON names.
// If multiple fields all have the same JSON name, then the one
// at shallowest depth takes precedence and the other fields at deeper depths
// are excluded from the list of JSON representable fields.
// If multiple fields at the shallowest depth have the same JSON name,
// but exactly one is explicitly tagged with a JSON name,
// then that field takes precedence and all others are excluded from the list.
// This is analogous to Go visibility rules for struct field selection
// with embedded struct types.
//
// Marshaling or unmarshaling a non-empty struct
// without any JSON representable fields results in a [SemanticError].
// Unexported fields must not have any `json` tags except for `json:"-"`.
//
// # Security Considerations
//
// JSON is frequently used as a data interchange format to communicate
// between different systems, possibly implemented in different languages.
// For interoperability and security reasons, it is important that
// all implementations agree upon the semantic meaning of the data.
//
// [For example, suppose we have two micro-services.]
// The first service is responsible for authenticating a JSON request,
// while the second service is responsible for executing the request
// (having assumed that the prior service authenticated the request).
// If an attacker were able to maliciously craft a JSON request such that
// both services believe that the same request is from different users,
// it could bypass the authenticator with valid credentials for one user,
// but maliciously perform an action on behalf of a different user.
//
// According to RFC 8259, there unfortunately exist many JSON texts
// that are syntactically valid but semantically ambiguous.
// For example, the standard does not define how to interpret duplicate
// names within an object.
//
// The v1 [encoding/json] and [encoding/json/v2] packages
// interpret some inputs in different ways. In particular:
//
// - The standard specifies that JSON must be encoded using UTF-8.
// By default, v1 replaces invalid bytes of UTF-8 in JSON strings
// with the Unicode replacement character,
// while v2 rejects inputs with invalid UTF-8.
// To change the default, specify the [jsontext.AllowInvalidUTF8] option.
// The replacement of invalid UTF-8 is a form of data corruption
// that alters the precise meaning of strings.
//
// - The standard does not specify a particular behavior when
// duplicate names are encountered within a JSON object,
// which means that different implementations may behave differently.
// By default, v1 allows for the presence of duplicate names,
// while v2 rejects duplicate names.
// To change the default, specify the [jsontext.AllowDuplicateNames] option.
// If allowed, object members are processed in the order they are observed,
// meaning that later values will replace or be merged into prior values,
// depending on the Go value type.
//
// - The standard defines a JSON object as an unordered collection of name/value pairs.
// While ordering can be observed through the underlying [jsontext] API,
// both v1 and v2 generally avoid exposing the ordering.
// No application should semantically depend on the order of object members.
// Allowing duplicate names is a vector through which ordering of members
// can accidentally be observed and depended upon.
//
// - The standard suggests that JSON object names are typically compared
// based on equality of the sequence of Unicode code points,
// which implies that comparing names is often case-sensitive.
// When unmarshaling a JSON object into a Go struct,
// by default, v1 uses a (loose) case-insensitive match on the name,
// while v2 uses a (strict) case-sensitive match on the name.
// To change the default, specify the [MatchCaseInsensitiveNames] option.
// The use of case-insensitive matching provides another vector through
// which duplicate names can occur. Allowing case-insensitive matching
// means that v1 or v2 might interpret JSON objects differently from most
// other JSON implementations (which typically use a case-sensitive match).
//
// - The standard does not specify a particular behavior when
// an unknown name in a JSON object is encountered.
// When unmarshaling a JSON object into a Go struct, by default
// both v1 and v2 ignore unknown names and their corresponding values.
// To change the default, specify the [RejectUnknownMembers] option.
//
// - The standard suggests that implementations may use a float64
// to represent a JSON number. Consequently, large JSON integers
// may lose precision when stored as a floating-point type.
// Both v1 and v2 correctly preserve precision when marshaling and
// unmarshaling a concrete integer type. However, even if v1 and v2
// preserve precision for concrete types, other JSON implementations
// may not be able to preserve precision for outputs produced by v1 or v2.
// The `string` tag option can be used to specify that an integer type
// is to be quoted within a JSON string to avoid loss of precision.
// Furthermore, v1 and v2 may still lose precision when unmarshaling
// into an any interface value, where unmarshal uses a float64
// by default to represent a JSON number.
// To change the default, specify the [WithUnmarshalers] option
// with a custom unmarshaler that pre-populates the interface value
// with a concrete Go type that can preserve precision.
//
// RFC 8785 specifies a canonical form for any JSON text,
// which explicitly defines specific behaviors that RFC 8259 leaves undefined.
// In theory, if a text can successfully [jsontext.Value.Canonicalize]
// without changing the semantic meaning of the data, then it provides a
// greater degree of confidence that the data is more secure and interoperable.
//
// The v2 API generally chooses more secure defaults than v1,
// but care should still be taken with large integers or unknown members.
//
// [For example, suppose we have two micro-services.]: https://www.youtube.com/watch?v=avilmOcHKHE&t=1057s
package json
// requireKeyedLiterals can be embedded in a struct to require keyed literals.
type requireKeyedLiterals struct{}
// nonComparable can be embedded in a struct to prevent comparability.
type nonComparable [0]func()

View File

@@ -0,0 +1,465 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"cmp"
"errors"
"fmt"
"io"
"reflect"
"strconv"
"strings"
"sync"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
"github.com/go-json-experiment/json/jsontext"
)
// ErrUnknownName indicates that a JSON object member could not be
// unmarshaled because the name is not known to the target Go struct.
// This error is directly wrapped within a [SemanticError] when produced.
//
// The name of an unknown JSON object member can be extracted as:
//
// err := ...
// serr, ok := errors.AsType[*json.SemanticError](err)
// if ok && serr.Err == json.ErrUnknownName {
// ptr := serr.JSONPointer // JSON pointer to unknown name
// name := ptr.LastToken() // unknown name itself
// ...
// }
//
// This error is only returned if [RejectUnknownMembers] is true.
var ErrUnknownName = errors.New("unknown object member name")
// errAmbiguousName indicates a JSON object member could be unmarshal
// into multiple candidate Go struct fields.
var errAmbiguousName = errors.New("ambiguous object member name")
const errorPrefix = "json: "
func isSemanticError(err error) bool {
_, ok := err.(*SemanticError)
return ok
}
func isSyntacticError(err error) bool {
_, ok := err.(*jsontext.SyntacticError)
return ok
}
// isFatalError reports whether this error must terminate arshaling.
// All errors are considered fatal unless operating under
// [jsonflags.ReportErrorsWithLegacySemantics] in which case only
// syntactic errors and I/O errors are considered fatal.
func isFatalError(err error, flags jsonflags.Flags) bool {
return !flags.Get(jsonflags.ReportErrorsWithLegacySemantics) ||
isSyntacticError(err) || export.IsIOError(err)
}
// SemanticError describes an error determining the meaning
// of JSON data as Go data or vice-versa.
//
// If a [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom] method
// returns a SemanticError when called by the [json] package,
// then the ByteOffset, JSONPointer, and GoType fields are automatically
// populated by the calling context if they are the zero value.
//
// The contents of this error as produced by this package may change over time.
type SemanticError struct {
requireKeyedLiterals
nonComparable
action string // either "marshal" or "unmarshal"
// ByteOffset indicates that an error occurred at or after this byte offset.
ByteOffset int64
// JSONPointer indicates that an error occurred within this JSON value
// as indicated using the JSON Pointer notation (see RFC 6901).
JSONPointer jsontext.Pointer
// JSONKind is the JSON kind that could not be handled.
JSONKind jsontext.Kind // may be zero if unknown
// JSONValue is the JSON number or string that could not be unmarshaled.
// It is not populated during marshaling.
JSONValue jsontext.Value // may be nil if irrelevant or unknown
// GoType is the Go type that could not be handled.
GoType reflect.Type // may be nil if unknown
// Err is the underlying error.
Err error // may be nil
}
// coder is implemented by [jsontext.Encoder] or [jsontext.Decoder].
type coder interface {
StackPointer() jsontext.Pointer
Options() Options
}
// newInvalidFormatError constructs a SemanticError because
// the current type t cannot handle the provided options format.
// This function must be called before producing or consuming the next value.
//
// If [jsonflags.ReportErrorsWithLegacySemantics] is specified,
// then this automatically skips the next value when unmarshaling
// to ensure that the value is fully consumed.
func newInvalidFormatError(c coder, t reflect.Type) error {
err := fmt.Errorf("invalid format flag %q", c.Options().(*jsonopts.Struct).Format)
switch c := c.(type) {
case *jsontext.Encoder:
err = newMarshalErrorBefore(c, t, err)
case *jsontext.Decoder:
err = newUnmarshalErrorBeforeWithSkipping(c, t, err)
}
return err
}
// newMarshalErrorBefore wraps err in a SemanticError assuming that e
// is positioned right before the next token or value, which causes an error.
func newMarshalErrorBefore(e *jsontext.Encoder, t reflect.Type, err error) error {
return &SemanticError{action: "marshal", GoType: t, Err: toUnexpectedEOF(err),
ByteOffset: e.OutputOffset() + int64(export.Encoder(e).CountNextDelimWhitespace()),
JSONPointer: jsontext.Pointer(export.Encoder(e).AppendStackPointer(nil, +1))}
}
// newUnmarshalErrorBefore wraps err in a SemanticError assuming that d
// is positioned right before the next token or value, which causes an error.
// It does not record the next JSON kind as this error is used to indicate
// the receiving Go value is invalid to unmarshal into (and not a JSON error).
// However, if [jsonflags.ReportErrorsWithLegacySemantics] is specified,
// then it does record the next JSON kind for historical reporting reasons.
func newUnmarshalErrorBefore(d *jsontext.Decoder, t reflect.Type, err error) error {
var k jsontext.Kind
if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
k = d.PeekKind()
}
return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err),
ByteOffset: d.InputOffset() + int64(export.Decoder(d).CountNextDelimWhitespace()),
JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, +1)),
JSONKind: k}
}
// newUnmarshalErrorBeforeWithSkipping is like [newUnmarshalErrorBefore],
// but automatically skips the next value if
// [jsonflags.ReportErrorsWithLegacySemantics] is specified.
func newUnmarshalErrorBeforeWithSkipping(d *jsontext.Decoder, t reflect.Type, err error) error {
err = newUnmarshalErrorBefore(d, t, err)
if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
if err2 := export.Decoder(d).SkipValue(); err2 != nil {
return err2
}
}
return err
}
// newUnmarshalErrorAfter wraps err in a SemanticError assuming that d
// is positioned right after the previous token or value, which caused an error.
func newUnmarshalErrorAfter(d *jsontext.Decoder, t reflect.Type, err error) error {
tokOrVal := export.Decoder(d).PreviousTokenOrValue()
byteOffset := d.InputOffset() - int64(len(tokOrVal))
if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
// NOTE: In v1, the offset pointed to the end of the bad token or value.
if k := jsontext.Value(tokOrVal).Kind(); k == '[' || k == '{' {
byteOffset++ // add just the '[' or '{'
} else {
byteOffset += int64(len(tokOrVal))
}
}
return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err),
ByteOffset: byteOffset,
JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, -1)),
JSONKind: jsontext.Value(tokOrVal).Kind()}
}
// newUnmarshalErrorAfterWithValue wraps err in a SemanticError assuming that d
// is positioned right after the previous token or value, which caused an error.
// It also stores a copy of the last JSON value if it is a string or number.
func newUnmarshalErrorAfterWithValue(d *jsontext.Decoder, t reflect.Type, err error) error {
serr := newUnmarshalErrorAfter(d, t, err).(*SemanticError)
if serr.JSONKind == '"' || serr.JSONKind == '0' {
serr.JSONValue = jsontext.Value(export.Decoder(d).PreviousTokenOrValue()).Clone()
}
return serr
}
// newUnmarshalErrorAfterWithSkipping is like [newUnmarshalErrorAfter],
// but automatically skips the remainder of the current value if
// [jsonflags.ReportErrorsWithLegacySemantics] is specified.
func newUnmarshalErrorAfterWithSkipping(d *jsontext.Decoder, t reflect.Type, err error) error {
err = newUnmarshalErrorAfter(d, t, err)
if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) {
if err2 := export.Decoder(d).SkipValueRemainder(); err2 != nil {
return err2
}
}
return err
}
// newSemanticErrorWithPosition wraps err in a SemanticError assuming that
// the error occurred at the provided depth, and length.
// If err is already a SemanticError, then position information is only
// injected if it is currently unpopulated.
//
// If the position is unpopulated, it is ambiguous where the error occurred
// in the user code, whether it was before or after the current position.
// For the byte offset, we assume that the error occurred before the last read
// token or value when decoding, or before the next value when encoding.
// For the JSON pointer, we point to the parent object or array unless
// we can be certain that it happened with an object member.
//
// This is used to annotate errors returned by user-provided
// v2 MarshalJSON or UnmarshalJSON methods or functions.
func newSemanticErrorWithPosition(c coder, t reflect.Type, prevDepth int, prevLength int64, err error) error {
serr, _ := err.(*SemanticError)
if serr == nil {
serr = &SemanticError{Err: err}
}
serr.Err = toUnexpectedEOF(serr.Err)
var currDepth int
var currLength int64
var coderState interface{ AppendStackPointer([]byte, int) []byte }
var offset int64
switch c := c.(type) {
case *jsontext.Encoder:
e := export.Encoder(c)
serr.action = cmp.Or(serr.action, "marshal")
currDepth, currLength = e.Tokens.DepthLength()
offset = c.OutputOffset() + int64(export.Encoder(c).CountNextDelimWhitespace())
coderState = e
case *jsontext.Decoder:
d := export.Decoder(c)
serr.action = cmp.Or(serr.action, "unmarshal")
currDepth, currLength = d.Tokens.DepthLength()
tokOrVal := d.PreviousTokenOrValue()
offset = c.InputOffset() - int64(len(tokOrVal))
if (prevDepth == currDepth && prevLength == currLength) || len(tokOrVal) == 0 {
// If no Read method was called in the user-defined method or
// if the Peek method was called, then use the offset of the next value.
offset = c.InputOffset() + int64(export.Decoder(c).CountNextDelimWhitespace())
}
coderState = d
}
serr.ByteOffset = cmp.Or(serr.ByteOffset, offset)
if serr.JSONPointer == "" {
where := 0 // default to ambiguous positioning
switch {
case prevDepth == currDepth && prevLength+0 == currLength:
where = +1
case prevDepth == currDepth && prevLength+1 == currLength:
where = -1
}
serr.JSONPointer = jsontext.Pointer(coderState.AppendStackPointer(nil, where))
}
serr.GoType = cmp.Or(serr.GoType, t)
return serr
}
// collapseSemanticErrors collapses double SemanticErrors at the outer levels
// into a single SemanticError by preserving the inner error,
// but prepending the ByteOffset and JSONPointer with the outer error.
//
// For example:
//
// collapseSemanticErrors(&SemanticError{
// ByteOffset: len64(`[0,{"alpha":[0,1,`),
// JSONPointer: "/1/alpha/2",
// GoType: reflect.TypeFor[outerType](),
// Err: &SemanticError{
// ByteOffset: len64(`{"foo":"bar","fizz":[0,`),
// JSONPointer: "/fizz/1",
// GoType: reflect.TypeFor[innerType](),
// Err: ...,
// },
// })
//
// results in:
//
// &SemanticError{
// ByteOffset: len64(`[0,{"alpha":[0,1,`) + len64(`{"foo":"bar","fizz":[0,`),
// JSONPointer: "/1/alpha/2" + "/fizz/1",
// GoType: reflect.TypeFor[innerType](),
// Err: ...,
// }
//
// This is used to annotate errors returned by user-provided
// v1 MarshalJSON or UnmarshalJSON methods with precise position information
// if they themselves happened to return a SemanticError.
// Since MarshalJSON and UnmarshalJSON are not operating on the root JSON value,
// their positioning must be relative to the nested JSON value
// returned by UnmarshalJSON or passed to MarshalJSON.
// Therefore, we can construct an absolute position by concatenating
// the outer with the inner positions.
//
// Note that we do not use collapseSemanticErrors with user-provided functions
// that take in an [jsontext.Encoder] or [jsontext.Decoder] since they contain
// methods to report position relative to the root JSON value.
// We assume user-constructed errors are correctly precise about position.
func collapseSemanticErrors(err error) error {
if serr1, ok := err.(*SemanticError); ok {
if serr2, ok := serr1.Err.(*SemanticError); ok {
serr2.ByteOffset = serr1.ByteOffset + serr2.ByteOffset
serr2.JSONPointer = serr1.JSONPointer + serr2.JSONPointer
*serr1 = *serr2
}
}
return err
}
func wrapErrUnsupported(err error, what string) error {
if errors.Is(err, errors.ErrUnsupported) {
return errors.New(what + " may not return errors.ErrUnsupported")
}
return err
}
// errorModalVerb is a modal verb like "cannot" or "unable to".
//
// Once per process, Hyrum-proof the error message by deliberately
// switching between equivalent renderings of the same error message.
// The randomization is tied to the Hyrum-proofing already applied
// on map iteration in Go.
var errorModalVerb = sync.OnceValue(func() string {
for phrase := range map[string]struct{}{"cannot": {}, "unable to": {}} {
return phrase // use whichever phrase we get in the first iteration
}
return ""
})
func (e *SemanticError) Error() string {
var sb strings.Builder
sb.WriteString(errorPrefix)
sb.WriteString(errorModalVerb())
// Format action.
var preposition string
switch e.action {
case "marshal":
sb.WriteString(" marshal")
preposition = " from"
case "unmarshal":
sb.WriteString(" unmarshal")
preposition = " into"
default:
sb.WriteString(" handle")
preposition = " with"
}
// Format JSON kind.
switch e.JSONKind {
case 'n':
sb.WriteString(" JSON null")
case 'f', 't':
sb.WriteString(" JSON boolean")
case '"':
sb.WriteString(" JSON string")
case '0':
sb.WriteString(" JSON number")
case '{', '}':
sb.WriteString(" JSON object")
case '[', ']':
sb.WriteString(" JSON array")
default:
if e.action == "" {
preposition = ""
}
}
if len(e.JSONValue) > 0 && len(e.JSONValue) < 100 {
sb.WriteByte(' ')
sb.Write(e.JSONValue)
}
// Format Go type.
if e.GoType != nil {
typeString := e.GoType.String()
if len(typeString) > 100 {
// An excessively long type string most likely occurs for
// an anonymous struct declaration with many fields.
// Reduce the noise by just printing the kind,
// and optionally prepending it with the package name
// if the struct happens to include an unexported field.
typeString = e.GoType.Kind().String()
if e.GoType.Kind() == reflect.Struct && e.GoType.Name() == "" {
for i := range e.GoType.NumField() {
if pkgPath := e.GoType.Field(i).PkgPath; pkgPath != "" {
typeString = pkgPath[strings.LastIndexByte(pkgPath, '/')+len("/"):] + ".struct"
break
}
}
}
}
sb.WriteString(preposition)
sb.WriteString(" Go ")
sb.WriteString(typeString)
}
// Special handling for unknown names.
if e.Err == ErrUnknownName || e.Err == errAmbiguousName {
sb.WriteString(": ")
sb.WriteString(e.Err.Error())
sb.WriteString(" ")
sb.WriteString(strconv.Quote(e.JSONPointer.LastToken()))
if parent := e.JSONPointer.Parent(); parent != "" {
sb.WriteString(" within ")
sb.WriteString(strconv.Quote(jsonwire.TruncatePointer(string(parent), 100)))
}
return sb.String()
}
// Format where.
// Avoid printing if it overlaps with a wrapped SyntacticError.
switch serr, _ := e.Err.(*jsontext.SyntacticError); {
case e.JSONPointer != "":
if serr == nil || !e.JSONPointer.Contains(serr.JSONPointer) {
sb.WriteString(" within ")
sb.WriteString(strconv.Quote(jsonwire.TruncatePointer(string(e.JSONPointer), 100)))
}
case e.ByteOffset > 0:
if serr == nil || !(e.ByteOffset <= serr.ByteOffset) {
sb.WriteString(" after offset ")
sb.WriteString(strconv.FormatInt(e.ByteOffset, 10))
}
}
// Format underlying error.
if e.Err != nil {
errString := e.Err.Error()
if isSyntacticError(e.Err) {
errString = strings.TrimPrefix(errString, "jsontext: ")
}
sb.WriteString(": ")
sb.WriteString(errString)
}
return sb.String()
}
func (e *SemanticError) Unwrap() error {
return e.Err
}
func newDuplicateNameError(ptr jsontext.Pointer, quotedName []byte, offset int64) error {
if quotedName != nil {
name, _ := jsonwire.AppendUnquote(nil, quotedName)
ptr = ptr.AppendToken(string(name))
}
return &jsontext.SyntacticError{
ByteOffset: offset,
JSONPointer: ptr,
Err: jsontext.ErrDuplicateName,
}
}
// toUnexpectedEOF converts [io.EOF] to [io.ErrUnexpectedEOF].
func toUnexpectedEOF(err error) error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}

View File

@@ -0,0 +1,656 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"cmp"
"errors"
"fmt"
"io"
"reflect"
"slices"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
type isZeroer interface {
IsZero() bool
}
var isZeroerType = reflect.TypeFor[isZeroer]()
type structFields struct {
flattened []structField // listed in depth-first ordering
byActualName map[string]*structField
byFoldedName map[string][]*structField
embeddedFallback *structField
errUnsupportedFormat *SemanticError
}
// reindex recomputes index to avoid bounds check during runtime.
//
// During the construction of each [structField] in [makeStructFields],
// the index field is 0-indexed. However, before it returns,
// the 0th field is stored in index0 and index stores the remainder.
func (sf *structFields) reindex() {
reindex := func(f *structField) {
f.index0 = f.index[0]
f.index = f.index[1:]
if len(f.index) == 0 {
f.index = nil // avoid pinning the backing slice
}
}
for i := range sf.flattened {
reindex(&sf.flattened[i])
}
if sf.embeddedFallback != nil {
reindex(sf.embeddedFallback)
}
}
// lookupByFoldedName looks up name by a case-insensitive match
// that also ignores the presence of dashes and underscores.
func (fs *structFields) lookupByFoldedName(name []byte) []*structField {
return fs.byFoldedName[string(foldName(name))]
}
type structField struct {
id int // unique numeric ID in breadth-first ordering
index0 int // 0th index into a struct according to [reflect.Type.FieldByIndex]
index []int // 1st index and remainder according to [reflect.Type.FieldByIndex]
typ reflect.Type
fncs *arshaler
isZero func(addressableValue) bool
isEmpty func(addressableValue) bool
fieldOptions
}
var errNoExportedFields = errors.New("Go struct has no exported fields")
func makeStructFields(root reflect.Type) (fs structFields, serr *SemanticError) {
orErrorf := func(serr *SemanticError, t reflect.Type, f string, a ...any) *SemanticError {
return cmp.Or(serr, &SemanticError{GoType: t, Err: fmt.Errorf(f, a...)})
}
// Setup a queue for a breadth-first search.
var queueIndex int
type queueEntry struct {
typ reflect.Type
index []int
visitChildren bool // whether to recursively visit embedded field in this struct
}
queue := []queueEntry{{root, nil, true}}
seen := map[reflect.Type]bool{root: true}
// Perform a breadth-first search over all reachable fields.
// This ensures that len(f.index) will be monotonically increasing.
var allFields, embeddedFallbacks []structField
for queueIndex < len(queue) {
qe := queue[queueIndex]
queueIndex++
t := qe.typ
embeddedFallbackIndex := -1 // index of last embedded fallback field in current struct
namesIndex := make(map[string]int) // index of each field with a given JSON object name in current struct
var hasAnyJSONTag bool // whether any Go struct field has a `json` tag
var hasAnyJSONField bool // whether any JSON serializable fields exist in current struct
for i := range t.NumField() {
sf := t.Field(i)
_, hasTag := sf.Tag.Lookup("json")
hasAnyJSONTag = hasAnyJSONTag || hasTag
options, ignored, err := parseFieldOptions(sf)
if err != nil {
serr = cmp.Or(serr, &SemanticError{GoType: t, Err: err})
}
if ignored {
continue
}
hasAnyJSONField = true
f := structField{
// Allocate a new slice (len=N+1) to hold both
// the parent index (len=N) and the current index (len=1).
// Do this to avoid clobbering the memory of the parent index.
index: append(append(make([]int, 0, len(qe.index)+1), qe.index...), i),
typ: sf.Type,
fieldOptions: options,
}
if sf.Anonymous && !f.hasName {
if indirectType(f.typ).Kind() != reflect.Struct {
serr = orErrorf(serr, t, "embedded Go struct field %s of non-struct type must be explicitly given a JSON name", sf.Name)
} else {
f.embed = true // implied by use of Go embedding without an explicit name
}
}
var handleEmbed, handleField func()
handleEmbed = func() {
// Handle an embedded field that serializes to/from
// zero or more JSON object members.
if f.fieldOptions != (fieldOptions{name: f.name, quotedName: f.quotedName, embed: true}) {
serr = orErrorf(serr, t, "Go struct field %s cannot have any options other than `embed` specified", sf.Name)
if f.hasName {
handleField()
return // invalid embedded field; treat as regular field
}
f.fieldOptions = fieldOptions{name: f.name, quotedName: f.quotedName, embed: f.embed}
}
// Reject any types with custom serialization otherwise
// it becomes impossible to know what sub-fields to embed.
tf := indirectType(f.typ)
if implementsAny(tf, allMethodTypes...) && tf != jsontextValueType {
serr = orErrorf(serr, t, "embedded Go struct field %s of type %s must not implement marshal or unmarshal methods", sf.Name, tf)
}
// Handle an embedded field that serializes to/from
// a finite number of JSON object members backed by a Go struct.
if tf.Kind() == reflect.Struct {
if qe.visitChildren {
queue = append(queue, queueEntry{tf, f.index, !seen[tf]})
}
seen[tf] = true
return
} else if !sf.IsExported() {
serr = orErrorf(serr, t, "embedded Go struct field %s is not exported", sf.Name)
return // invalid embedded field; treat as ignored
}
// Handle an embedded field that serializes to/from any number of
// JSON object members back by a Go map or jsontext.Value.
switch {
case tf == jsontextValueType:
f.fncs = nil // specially handled in arshal_embedded.go
case tf.Kind() == reflect.Map && tf.Key().Kind() == reflect.String:
if implementsAny(tf.Key(), allMethodTypes...) {
serr = orErrorf(serr, t, "embedded map field %s of type %s must have a string key that does not implement marshal or unmarshal methods", sf.Name, tf)
handleField()
return // invalid embedded field; treat as regular field
}
f.fncs = lookupArshaler(tf.Elem())
default:
serr = orErrorf(serr, t, "embedded Go struct field %s of type %s must be a Go struct, Go map of string key, or jsontext.Value", sf.Name, tf)
handleField()
return // invalid embedded field; treat as regular field
}
// Reject multiple embedded fallback fields within the same struct.
if embeddedFallbackIndex >= 0 {
serr = orErrorf(serr, t, "embedded Go struct fields %s and %s cannot both be a Go map or jsontext.Value", t.Field(embeddedFallbackIndex).Name, sf.Name)
// Still append f to embeddedFallbacks as there is still a
// check for a dominant embedded fallback before returning.
}
embeddedFallbackIndex = i
embeddedFallbacks = append(embeddedFallbacks, f)
}
handleField = func() {
// Handle normal Go struct field that serializes to/from
// a single JSON object member.
// Unexported fields cannot be serialized except for
// embedded fields of a struct type,
// which might promote exported fields of their own.
if !sf.IsExported() {
tf := indirectType(f.typ)
if !(sf.Anonymous && tf.Kind() == reflect.Struct) {
serr = orErrorf(serr, t, "Go struct field %s is not exported", sf.Name)
return
}
// Unfortunately, methods on the unexported field
// still cannot be called.
if implementsAny(tf, allMethodTypes...) ||
(f.omitzero && implementsAny(tf, isZeroerType)) {
serr = orErrorf(serr, t, "Go struct field %s is not exported for method calls", sf.Name)
return
}
}
// Provide a function that uses a type's IsZero method.
switch {
case sf.Type.Kind() == reflect.Interface && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on a nil interface or
// non-nil interface with nil pointer.
return va.IsNil() || (va.Elem().Kind() == reflect.Pointer && va.Elem().IsNil()) || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Kind() == reflect.Pointer && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on nil pointer.
return va.IsNil() || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Interface().(isZeroer).IsZero() }
case reflect.PointerTo(sf.Type).Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Addr().Interface().(isZeroer).IsZero() }
}
// Provide a function that can determine whether the value would
// serialize as an empty JSON value.
switch sf.Type.Kind() {
case reflect.String, reflect.Map, reflect.Array, reflect.Slice:
f.isEmpty = func(va addressableValue) bool { return va.Len() == 0 }
case reflect.Pointer, reflect.Interface:
f.isEmpty = func(va addressableValue) bool { return va.IsNil() }
}
// Reject multiple fields with same name within the same struct.
if j, ok := namesIndex[f.name]; ok {
serr = orErrorf(serr, t, "Go struct fields %s and %s conflict over JSON object name %q", t.Field(j).Name, sf.Name, f.name)
// Still append f to allFields as there is still a
// check for a dominant field before returning.
}
namesIndex[f.name] = i
f.id = len(allFields)
f.fncs = lookupArshaler(sf.Type)
allFields = append(allFields, f)
if f.format != "" && fs.errUnsupportedFormat == nil {
fs.errUnsupportedFormat = &SemanticError{GoType: t, Err: fmt.Errorf("Go struct field %s has unsupported `format` tag option", sf.Name)}
}
}
if f.embed {
handleEmbed()
} else {
handleField()
}
}
// NOTE: New users to the json package are occasionally surprised that
// unexported fields are ignored. This occurs by necessity due to our
// inability to directly introspect such fields with Go reflection
// without the use of unsafe.
//
// To reduce friction here, refuse to serialize any Go struct that
// has no JSON serializable fields, has at least one Go struct field,
// and does not have any `json` tags present. For example,
// errors returned by errors.New would fail to serialize.
isEmptyStruct := t.NumField() == 0
if !isEmptyStruct && !hasAnyJSONTag && !hasAnyJSONField {
serr = cmp.Or(serr, &SemanticError{GoType: t, Err: errNoExportedFields})
}
}
// Sort the fields by exact name (breaking ties by depth and
// then by presence of an explicitly provided JSON name).
// Select the dominant field from each set of fields with the same name.
// If multiple fields have the same name, then the dominant field
// is the one that exists alone at the shallowest depth,
// or the one that is uniquely tagged with a JSON name.
// Otherwise, no dominant field exists for the set.
flattened := allFields[:0]
slices.SortStableFunc(allFields, func(x, y structField) int {
return cmp.Or(
strings.Compare(x.name, y.name),
cmp.Compare(len(x.index), len(y.index)),
boolsCompare(!x.hasName, !y.hasName))
})
for len(allFields) > 0 {
n := 1 // number of fields with the same exact name
for n < len(allFields) && allFields[n-1].name == allFields[n].name {
n++
}
if n == 1 || len(allFields[0].index) != len(allFields[1].index) || allFields[0].hasName != allFields[1].hasName {
flattened = append(flattened, allFields[0]) // only keep field if there is a dominant field
}
allFields = allFields[n:]
}
// Sort the fields according to a breadth-first ordering
// so that we can re-number IDs with the smallest possible values.
// This optimizes use of uintSet such that it fits in the 64-entry bit set.
slices.SortFunc(flattened, func(x, y structField) int {
return cmp.Compare(x.id, y.id)
})
for i := range flattened {
flattened[i].id = i
}
// Sort the fields according to a depth-first ordering
// as the typical order that fields are marshaled.
slices.SortFunc(flattened, func(x, y structField) int {
return slices.Compare(x.index, y.index)
})
// Compute the mapping of fields in the byActualName map.
// Pre-fold all names so that we can lookup folded names quickly.
fs = structFields{
flattened: flattened,
byActualName: make(map[string]*structField, len(flattened)),
byFoldedName: make(map[string][]*structField, len(flattened)),
errUnsupportedFormat: fs.errUnsupportedFormat,
}
for i, f := range fs.flattened {
foldedName := string(foldName([]byte(f.name)))
fs.byActualName[f.name] = &fs.flattened[i]
fs.byFoldedName[foldedName] = append(fs.byFoldedName[foldedName], &fs.flattened[i])
}
for foldedName, fields := range fs.byFoldedName {
if len(fields) > 1 {
// The precedence order for conflicting case-insensitive names
// is by breadth-first order, rather than depth-first order.
slices.SortFunc(fields, func(x, y *structField) int {
return cmp.Compare(x.id, y.id)
})
fs.byFoldedName[foldedName] = fields
}
}
if n := len(embeddedFallbacks); n == 1 || (n > 1 && len(embeddedFallbacks[0].index) != len(embeddedFallbacks[1].index)) {
fs.embeddedFallback = &embeddedFallbacks[0] // dominant embedded fallback field
}
fs.reindex()
return fs, serr
}
// indirectType unwraps one level of pointer indirection
// similar to how Go only allows embedding either T or *T,
// but not **T or P (which is a named pointer).
func indirectType(t reflect.Type) reflect.Type {
if t.Kind() == reflect.Pointer && t.Name() == "" {
t = t.Elem()
}
return t
}
// matchFoldedName matches a case-insensitive name depending on the options.
// It assumes that foldName(f.name) == foldName(name).
//
// Case-insensitive matching is used if the `case:ignore` tag option is specified
// or the MatchCaseInsensitiveNames call option is specified
// (and the `case:strict` tag option is not specified).
// Functionally, the `case:ignore` and `case:strict` tag options take precedence.
//
// The v1 definition of case-insensitivity operated under strings.EqualFold
// and would strictly compare dashes and underscores,
// while the v2 definition would ignore the presence of dashes and underscores.
// Thus, if the MatchCaseSensitiveDelimiter call option is specified,
// the match is further restricted to using strings.EqualFold.
func (f *structField) matchFoldedName(name []byte, flags *jsonflags.Flags) bool {
if f.casing == caseIgnore || (flags.Get(jsonflags.MatchCaseInsensitiveNames) && f.casing != caseStrict) {
if !flags.Get(jsonflags.MatchCaseSensitiveDelimiter) || strings.EqualFold(string(name), f.name) {
return true
}
}
return false
}
const (
caseIgnore = 1
caseStrict = 2
)
type fieldOptions struct {
name string
quotedName string // quoted name per RFC 8785, section 3.2.2.2.
hasName bool
nameNeedEscape bool
casing int8 // either 0, caseIgnore, or caseStrict
embed bool
omitzero bool
omitempty bool
string bool
format string
}
// parseFieldOptions parses the `json` tag in a Go struct field as
// a structured set of options configuring parameters such as
// the JSON member name and other features.
func parseFieldOptions(sf reflect.StructField) (out fieldOptions, ignored bool, err error) {
tag, hasTag := sf.Tag.Lookup("json")
// Check whether this field is explicitly ignored.
if tag == "-" {
return fieldOptions{}, true, nil
}
// Check whether this field is unexported and not embedded,
// which Go reflection cannot mutate for the sake of serialization.
//
// An embedded field of an unexported type is still capable of
// forwarding exported fields, which may be JSON serialized.
// This technically operates on the edge of what is permissible by
// the Go language, but the most recent decision is to permit this.
//
// See https://go.dev/issue/24153 and https://go.dev/issue/32772.
if !sf.IsExported() && !sf.Anonymous {
// Tag options specified on an unexported field suggests user error.
if hasTag {
err = cmp.Or(err, fmt.Errorf("unexported Go struct field %s cannot have non-ignored `json:%q` tag", sf.Name, tag))
}
return fieldOptions{}, true, err
}
// Determine the JSON member name for this Go field.
out.name = sf.Name // always starts with an uppercase character
if len(tag) > 0 && !strings.HasPrefix(tag, ",") {
// For better compatibility with v1, accept almost any unescaped name.
n := len(tag) - len(strings.TrimLeftFunc(tag, func(r rune) bool {
return !strings.ContainsRune(",\\'\"`", r) // reserve comma, backslash, and quotes
}))
name := tag[:n]
// If the next character is not a comma, then the name is either
// malformed (if n > 0) or a single-quoted name.
// In either case, call consumeTagOption to handle it further.
var err2 error
if !strings.HasPrefix(tag[n:], ",") && len(name) != len(tag) {
name, n, err2 = consumeTagOption(tag, false)
if err2 != nil {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err2))
}
}
if !utf8.ValidString(name) {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has JSON object name %q with invalid UTF-8", sf.Name, name))
name = string([]rune(name)) // replace invalid UTF-8 with utf8.RuneError
}
if err2 == nil {
out.hasName = true
out.name = name
}
tag = tag[n:]
}
b, _ := jsonwire.AppendQuote(nil, []byte(out.name), &jsonflags.Flags{})
out.quotedName = string(b)
out.nameNeedEscape = jsonwire.NeedEscape([]byte(out.name))
// Handle any additional tag options (if any).
var wasFormat bool
seenOpts := make(map[string]bool)
for len(tag) > 0 {
// Consume comma delimiter.
if tag[0] != ',' {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid character %q before next option (expecting ',')", sf.Name, tag[0]))
} else {
tag = tag[len(","):]
if len(tag) == 0 {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid trailing ',' character", sf.Name))
break
}
}
// Consume and process the tag option.
opt, n, err2 := consumeTagOption(tag, false)
if err2 != nil {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err2))
}
rawOpt := tag[:n]
tag = tag[n:]
switch {
case wasFormat:
err = cmp.Or(err, fmt.Errorf("Go struct field %s has `format` tag option that was not specified last", sf.Name))
case strings.HasPrefix(rawOpt, "'") && strings.TrimFunc(opt, isLetterOrDigit) == "":
err = cmp.Or(err, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `%s` tag option; specify `%s` instead", sf.Name, rawOpt, opt))
}
switch opt {
case "case":
if !strings.HasPrefix(tag, ":") {
err = cmp.Or(err, fmt.Errorf("Go struct field %s is missing value for `case` tag option; specify `case:ignore` or `case:strict` instead", sf.Name))
break
}
tag = tag[len(":"):]
opt, n, err2 := consumeTagOption(tag, false)
if err2 != nil {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed value for `case` tag option: %v", sf.Name, err2))
break
}
rawOpt := tag[:n]
tag = tag[n:]
if strings.HasPrefix(rawOpt, "'") {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `case:%s` tag option; specify `case:%s` instead", sf.Name, rawOpt, opt))
}
switch opt {
case "ignore":
out.casing |= caseIgnore
case "strict":
out.casing |= caseStrict
default:
err = cmp.Or(err, fmt.Errorf("Go struct field %s has unknown `case:%s` tag value", sf.Name, rawOpt))
}
case "embed":
out.embed = true
case "omitzero":
out.omitzero = true
case "omitempty":
out.omitempty = true
case "string":
out.string = true
case "format":
if !strings.HasPrefix(tag, ":") {
err = cmp.Or(err, fmt.Errorf("Go struct field %s is missing value for `format` tag option", sf.Name))
break
}
tag = tag[len(":"):]
opt, n, err2 := consumeTagOption(tag, true)
if err2 != nil {
err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed value for `format` tag option: %v", sf.Name, err2))
break
} else if opt == "" {
err = cmp.Or(err, fmt.Errorf("Go struct field %s cannot have empty value for `format` tag option", sf.Name))
break
}
tag = tag[n:]
out.format = opt
wasFormat = true
default:
// Reject keys that resemble one of the supported options.
// This catches invalid mutants such as "omitEmpty" or "omit_empty".
normOpt := strings.ReplaceAll(strings.ToLower(opt), "_", "")
switch normOpt {
case "case", "embed", "omitzero", "omitempty", "string", "format":
err = cmp.Or(err, fmt.Errorf("Go struct field %s has invalid appearance of `%s` tag option; specify `%s` instead", sf.Name, opt, normOpt))
}
// NOTE: Everything else is ignored. This does not mean it is
// forward compatible to insert arbitrary tag options since
// a future version of this package may understand that tag.
}
// Reject duplicates.
switch {
case out.casing == caseIgnore|caseStrict:
err = cmp.Or(err, fmt.Errorf("Go struct field %s cannot have both `case:ignore` and `case:strict` tag options", sf.Name))
case seenOpts[opt]:
err = cmp.Or(err, fmt.Errorf("Go struct field %s has duplicate appearance of `%s` tag option", sf.Name, rawOpt))
}
seenOpts[opt] = true
}
return out, false, err
}
// consumeTagOption consumes the next option,
// which is either a Go identifier or a single-quoted string.
// If the next option is invalid, it returns all of in until the next comma,
// and reports an error.
func consumeTagOption(in string, allowQuoted bool) (string, int, error) {
// For legacy compatibility with v1, assume options are comma-separated.
i := strings.IndexByte(in, ',')
if i < 0 {
i = len(in)
}
switch r, _ := utf8.DecodeRuneInString(in); {
// Option as a Go identifier.
case r == '_' || unicode.IsLetter(r):
n := len(in) - len(strings.TrimLeftFunc(in, isLetterOrDigit))
return in[:n], n, nil
// Option as a single-quoted string.
case r == '\'':
if !allowQuoted {
return in[:i], i, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter)", r)
}
// The grammar is nearly identical to a double-quoted Go string literal,
// but uses single quotes as the terminators. The reason for a custom
// grammar is because both backtick and double quotes cannot be used
// verbatim in a struct tag.
//
// Convert a single-quoted string to a double-quote string and rely on
// strconv.Unquote to handle the rest.
var inEscape bool
b := []byte{'"'}
n := len(`'`)
for len(in) > n {
r, rn := utf8.DecodeRuneInString(in[n:])
switch {
case inEscape:
if r == '\'' {
b = b[:len(b)-1] // remove escape character: `\'` => `'`
}
inEscape = false
case r == '\\':
inEscape = true
case r == '"':
b = append(b, '\\') // insert escape character: `"` => `\"`
case r == '\'':
b = append(b, '"')
n += len(`'`)
out, err := strconv.Unquote(string(b))
if err != nil {
return in[:i], i, fmt.Errorf("invalid single-quoted string: %s", in[:n])
}
return out, n, nil
}
b = append(b, in[n:][:rn]...)
n += rn
}
if n > 10 {
n = 10 // limit the amount of context printed in the error
}
return in[:i], i, fmt.Errorf("single-quoted string not terminated: %s...", in[:n])
case len(in) == 0:
return in[:i], i, io.ErrUnexpectedEOF
default:
if !allowQuoted {
return in[:i], i, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter)", r)
}
return in[:i], i, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter or single quote)", r)
}
}
func isLetterOrDigit(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
}
// boolsCompare compares x and y, ordering false before true.
func boolsCompare(x, y bool) int {
switch {
case !x && y:
return -1
default:
return 0
case x && !y:
return +1
}
}

View File

@@ -0,0 +1,58 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"unicode"
"unicode/utf8"
)
// foldName returns a folded string such that foldName(x) == foldName(y)
// is similar to strings.EqualFold(x, y), but ignores underscore and dashes.
// This allows foldName to match common naming conventions.
func foldName(in []byte) []byte {
// This is inlinable to take advantage of "function outlining".
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
var arr [32]byte // large enough for most JSON names
return appendFoldedName(arr[:0], in)
}
func appendFoldedName(out, in []byte) []byte {
for i := 0; i < len(in); {
// Handle single-byte ASCII.
if c := in[i]; c < utf8.RuneSelf {
if c != '_' && c != '-' {
if 'a' <= c && c <= 'z' {
c -= 'a' - 'A'
}
out = append(out, c)
}
i++
continue
}
// Handle multi-byte Unicode.
r, n := utf8.DecodeRune(in[i:])
out = utf8.AppendRune(out, foldRune(r))
i += n
}
return out
}
// foldRune is a variation on unicode.SimpleFold that returns the same rune
// for all runes in the same fold set.
//
// Invariant:
//
// foldRune(x) == foldRune(y) ⇔ strings.EqualFold(string(x), string(y))
func foldRune(r rune) rune {
for {
r2 := unicode.SimpleFold(r)
if r2 <= r {
return r2 // smallest character in the fold set
}
r = r2
}
}

View File

@@ -0,0 +1,88 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"encoding/binary"
"math/bits"
)
// stringCache is a cache for strings converted from a []byte.
type stringCache = [256]string // 256*unsafe.Sizeof(string("")) => 4KiB
// makeString returns the string form of b.
// It returns a pre-allocated string from c if present, otherwise
// it allocates a new string, inserts it into the cache, and returns it.
func makeString(c *stringCache, b []byte) string {
const (
minCachedLen = 2 // single byte strings are already interned by the runtime
maxCachedLen = 256 // large enough for UUIDs, IPv6 addresses, SHA-256 checksums, etc.
)
if c == nil || len(b) < minCachedLen || len(b) > maxCachedLen {
return string(b)
}
// Compute a hash from the fixed-width prefix and suffix of the string.
// This ensures hashing a string is a constant time operation.
var h uint32
switch {
case len(b) >= 8:
lo := binary.LittleEndian.Uint64(b[:8])
hi := binary.LittleEndian.Uint64(b[len(b)-8:])
h = hash64(uint32(lo), uint32(lo>>32)) ^ hash64(uint32(hi), uint32(hi>>32))
case len(b) >= 4:
lo := binary.LittleEndian.Uint32(b[:4])
hi := binary.LittleEndian.Uint32(b[len(b)-4:])
h = hash64(lo, hi)
case len(b) >= 2:
lo := binary.LittleEndian.Uint16(b[:2])
hi := binary.LittleEndian.Uint16(b[len(b)-2:])
h = hash64(uint32(lo), uint32(hi))
}
// Check the cache for the string.
i := h % uint32(len(*c))
if s := (*c)[i]; s == string(b) {
return s
}
s := string(b)
(*c)[i] = s
return s
}
// hash64 returns the hash of two uint32s as a single uint32.
func hash64(lo, hi uint32) uint32 {
// If avalanche=true, this is identical to XXH32 hash on an 8B string:
// var b [8]byte
// binary.LittleEndian.PutUint32(b[:4], lo)
// binary.LittleEndian.PutUint32(b[4:], hi)
// return xxhash.Sum32(b[:])
const (
prime1 = 0x9e3779b1
prime2 = 0x85ebca77
prime3 = 0xc2b2ae3d
prime4 = 0x27d4eb2f
prime5 = 0x165667b1
)
h := prime5 + uint32(8)
h += lo * prime3
h = bits.RotateLeft32(h, 17) * prime4
h += hi * prime3
h = bits.RotateLeft32(h, 17) * prime4
// Skip final mix (avalanche) step of XXH32 for performance reasons.
// Empirical testing shows that the improvements in unbiased distribution
// do not outweigh the extra cost in computational complexity.
const avalanche = false
if avalanche {
h ^= h >> 15
h *= prime2
h ^= h >> 13
h *= prime3
h ^= h >> 16
}
return h
}

View File

@@ -0,0 +1,42 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package internal
import "errors"
// NotForPublicUse is a marker type that an API is for internal use only.
// It does not perfectly prevent usage of that API, but helps to restrict usage.
// Anything with this marker is not covered by the Go compatibility agreement.
type NotForPublicUse struct{}
// AllowInternalUse is passed from "json" to "jsontext" to authenticate
// that the caller can have access to internal functionality.
var AllowInternalUse NotForPublicUse
// Sentinel error values internally shared between jsonv1 and jsonv2.
var (
ErrCycle = errors.New("encountered a cycle")
ErrNonNilReference = errors.New("value must be passed as a non-nil pointer reference")
ErrNilInterface = errors.New("cannot derive concrete type for nil interface with finite type set")
)
var (
// TransformMarshalError converts a v2 error into a v1 error.
// It is called only at the top-level of a Marshal function.
TransformMarshalError func(any, error) error
// NewMarshalerError constructs a jsonv1.MarshalerError.
// It is called after a user-defined Marshal method/function fails.
NewMarshalerError func(any, error, string) error
// TransformUnmarshalError converts a v2 error into a v1 error.
// It is called only at the top-level of an Unmarshal function.
TransformUnmarshalError func(any, error) error
// NewRawNumber returns new(jsonv1.Number).
NewRawNumber func() any
// RawNumberOf returns jsonv1.Number(b).
RawNumberOf func(b []byte) any
)

View File

@@ -0,0 +1,223 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
// jsonflags implements all the optional boolean flags.
// These flags are shared across both "json", "jsontext", and "jsonopts".
package jsonflags
import "github.com/go-json-experiment/json/internal"
// Bools represents zero or more boolean flags, all set to true or false.
// The least-significant bit is the boolean value of all flags in the set.
// The remaining bits identify which particular flags.
//
// In common usage, this is OR'd with 0 or 1. For example:
// - (AllowInvalidUTF8 | 0) means "AllowInvalidUTF8 is false"
// - (Multiline | Indent | 1) means "Multiline and Indent are true"
type Bools uint64
func (Bools) JSONOptions(internal.NotForPublicUse) {}
const (
// AllFlags is the set of all flags.
AllFlags = AllCoderFlags | AllArshalV2Flags | AllArshalV1Flags
// AllCoderFlags is the set of all encoder/decoder flags.
AllCoderFlags = (maxCoderFlag - 1) - initFlag
// AllArshalV2Flags is the set of all v2 marshal/unmarshal flags.
AllArshalV2Flags = (maxArshalV2Flag - 1) - (maxCoderFlag - 1)
// AllArshalV1Flags is the set of all v1 marshal/unmarshal flags.
AllArshalV1Flags = (maxArshalV1Flag - 1) - (maxArshalV2Flag - 1)
// NonBooleanFlags is the set of non-boolean flags,
// where the value is some other concrete Go type.
// The value of the flag is stored within jsonopts.Struct.
NonBooleanFlags = 0 |
Indent |
IndentPrefix |
ByteLimit |
DepthLimit |
Marshalers |
Unmarshalers |
FormatTag
// DefaultV1Flags is the set of boolean flags that default to true under
// v1 semantics. None of the non-boolean flags differ between v1 and v2.
DefaultV1Flags = 0 |
AllowDuplicateNames |
AllowInvalidUTF8 |
EscapeForHTML |
EscapeForJS |
PreserveRawStrings |
Deterministic |
FormatNilMapAsNull |
FormatNilSliceAsNull |
MatchCaseInsensitiveNames |
CallMethodsWithLegacySemantics |
FormatByteArrayAsArray |
FormatBytesWithLegacySemantics |
FormatDurationAsNano |
MatchCaseSensitiveDelimiter |
MergeWithLegacySemantics |
OmitEmptyWithLegacySemantics |
ParseBytesWithLooseRFC4648 |
ParseTimeWithLooseRFC3339 |
ReportErrorsWithLegacySemantics |
StringifyWithLegacySemantics |
UnmarshalArrayFromAnyLength
// AnyWhitespace reports whether the encoded output might have any whitespace.
AnyWhitespace = Multiline | SpaceAfterColon | SpaceAfterComma
// WhitespaceFlags is the set of flags related to whitespace formatting.
// In contrast to AnyWhitespace, this includes Indent and IndentPrefix
// as those settings take no effect if Multiline is false.
WhitespaceFlags = AnyWhitespace | Indent | IndentPrefix
// AnyEscape is the set of flags related to escaping in a JSON string.
AnyEscape = EscapeForHTML | EscapeForJS
// CanonicalizeNumbers is the set of flags related to raw number canonicalization.
CanonicalizeNumbers = CanonicalizeRawInts | CanonicalizeRawFloats
// TagFlags is the set of flags related to the presence of struct field tags.
// Tags have non-recursive effects, where the tag only applies to
// the top-level of the field value itself.
// Whenever descending into a JSON object or array, these flags are cleared.
TagFlags = StringTag | FormatTag
)
// Encoder and decoder flags.
const (
initFlag Bools = 1 << iota // reserved for the boolean value itself
AllowDuplicateNames // encode or decode
AllowInvalidUTF8 // encode or decode
WithinArshalCall // encode or decode; for internal use by json.Marshal and json.Unmarshal
OmitTopLevelNewline // encode only; for internal use by json.Marshal and json.MarshalWrite
PreserveRawStrings // encode only
CanonicalizeRawInts // encode only
CanonicalizeRawFloats // encode only
ReorderRawObjects // encode only
EscapeForHTML // encode only
EscapeForJS // encode only
Multiline // encode only
SpaceAfterColon // encode only
SpaceAfterComma // encode only
Indent // encode only; non-boolean flag
IndentPrefix // encode only; non-boolean flag
ByteLimit // encode or decode; non-boolean flag
DepthLimit // encode or decode; non-boolean flag
maxCoderFlag
)
// Marshal and Unmarshal flags (for v2).
const (
_ Bools = (maxCoderFlag >> 1) << iota
StringifyNumbers // marshal or unmarshal
Deterministic // marshal only
FormatNilMapAsNull // marshal only
FormatNilSliceAsNull // marshal only
OmitZeroStructFields // marshal only
MatchCaseInsensitiveNames // marshal or unmarshal
RejectUnknownMembers // unmarshal only
Marshalers // marshal only; non-boolean flag
Unmarshalers // unmarshal only; non-boolean flag
StringTag // marshal or unmarshal
FormatTag // marshal or unmarshal; non-boolean flag
FormatTagSupported // marshal or unmarshal
maxArshalV2Flag
)
// Marshal and Unmarshal flags (for v1).
const (
_ Bools = (maxArshalV2Flag >> 1) << iota
CallMethodsWithLegacySemantics // marshal or unmarshal
FormatByteArrayAsArray // marshal or unmarshal
FormatBytesWithLegacySemantics // marshal or unmarshal
FormatDurationAsNano // marshal or unmarshal
MatchCaseSensitiveDelimiter // marshal or unmarshal
MergeWithLegacySemantics // unmarshal
OmitEmptyWithLegacySemantics // marshal
ParseBytesWithLooseRFC4648 // unmarshal
ParseTimeWithLooseRFC3339 // unmarshal
ReportErrorsWithLegacySemantics // marshal or unmarshal
StringifyWithLegacySemantics // marshal or unmarshal
UnmarshalAnyWithRawNumber // unmarshal; for internal use by jsonv1.Decoder.UseNumber
UnmarshalArrayFromAnyLength // unmarshal
maxArshalV1Flag
)
// bitsUsed is the number of bits used in the 64-bit boolean flags
const bitsUsed = 43
// Static compile check that bitsUsed and maxArshalV1Flag are in sync.
const _ = uint64((1<<bitsUsed)-maxArshalV1Flag) + uint64(maxArshalV1Flag-(1<<bitsUsed))
// Flags is a set of boolean flags.
// If the presence bit is zero, then the value bit must also be zero.
// The least-significant bit of both fields is always zero.
//
// Unlike Bools, which can represent a set of bools that are all true or false,
// Flags represents a set of bools, each individually may be true or false.
type Flags struct{ Presence, Values uint64 }
// Join joins two sets of flags such that the latter takes precedence.
func (dst *Flags) Join(src Flags) {
// Copy over all source presence bits over to the destination (using OR),
// then invert the source presence bits to clear out source value (using AND-NOT),
// then copy over source value bits over to the destination (using OR).
// e.g., dst := Flags{Presence: 0b_1100_0011, Values: 0b_1000_0011}
// e.g., src := Flags{Presence: 0b_0101_1010, Values: 0b_1001_0010}
dst.Presence |= src.Presence // e.g., 0b_1100_0011 | 0b_0101_1010 -> 0b_1101_1011
dst.Values &= ^src.Presence // e.g., 0b_1000_0011 & 0b_1010_0101 -> 0b_1000_0001
dst.Values |= src.Values // e.g., 0b_1000_0001 | 0b_1001_0010 -> 0b_1001_0011
}
// Set sets both the presence and value for the provided bool (or set of bools).
func (fs *Flags) Set(f Bools) {
// Select out the bits for the flag identifiers (everything except LSB),
// then set the presence for all the identifier bits (using OR),
// then invert the identifier bits to clear out the values (using AND-NOT),
// then copy over all the identifier bits to the value if LSB is 1.
// e.g., fs := Flags{Presence: 0b_0101_0010, Values: 0b_0001_0010}
// e.g., f := 0b_1001_0001
id := uint64(f) &^ uint64(1) // e.g., 0b_1001_0001 & 0b_1111_1110 -> 0b_1001_0000
fs.Presence |= id // e.g., 0b_0101_0010 | 0b_1001_0000 -> 0b_1101_0010
fs.Values &= ^id // e.g., 0b_0001_0010 & 0b_0110_1111 -> 0b_0000_0010
fs.Values |= uint64(f&1) * id // e.g., 0b_0000_0010 | 0b_1001_0000 -> 0b_1001_0010
}
// Get reports whether the bool (or any of the bools) is true.
// This is generally only used with a singular bool.
// The value bit of f (i.e., the LSB) is ignored.
func (fs Flags) Get(f Bools) bool {
return fs.Values&uint64(f) > 0
}
// Has reports whether the bool (or any of the bools) is set.
// The value bit of f (i.e., the LSB) is ignored.
func (fs Flags) Has(f Bools) bool {
return fs.Presence&uint64(f) > 0
}
// Clear clears both the presence and value for the provided bool or bools.
// The value bit of f (i.e., the LSB) is ignored.
func (fs *Flags) Clear(f Bools) {
// Invert f to produce a mask to clear all bits in f (using AND).
// e.g., fs := Flags{Presence: 0b_0101_0010, Values: 0b_0001_0010}
// e.g., f := 0b_0001_1000
mask := uint64(^f) // e.g., 0b_0001_1000 -> 0b_1110_0111
fs.Presence &= mask // e.g., 0b_0101_0010 & 0b_1110_0111 -> 0b_0100_0010
fs.Values &= mask // e.g., 0b_0001_0010 & 0b_1110_0111 -> 0b_0000_0010
}

View File

@@ -0,0 +1,217 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsonopts
import (
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
)
// Options is the common options type shared across json packages.
type Options interface {
// JSONOptions is exported so related json packages can implement Options.
JSONOptions(internal.NotForPublicUse)
}
// experimentalFormatTagSupporter is a special option method that allows
// [github.com/go-json-experiment/json.ExperimentalSupportFormatTag]
// to enable experimental `format` tag functionality at runtime.
type experimentalFormatTagSupporter interface{ ExperimentalSupportFormatTag() bool }
// Struct is the combination of all options in struct form.
// This is efficient to pass down the call stack and to query.
type Struct struct {
Flags jsonflags.Flags
CoderValues
ArshalValues
}
type CoderValues struct {
Indent string // jsonflags.Indent
IndentPrefix string // jsonflags.IndentPrefix
ByteLimit int64 // jsonflags.ByteLimit
DepthLimit int // jsonflags.DepthLimit
}
type ArshalValues struct {
// The Marshalers and Unmarshalers fields use the any type to avoid a
// concrete dependency on *json.Marshalers and *json.Unmarshalers,
// which would in turn create a dependency on the "reflect" package.
Marshalers any // jsonflags.Marshalers
Unmarshalers any // jsonflags.Unmarshalers
Format string // valid if jsonflags.FormatTag is set
}
// DefaultOptionsV2 is the set of all options that define default v2 behavior.
var DefaultOptionsV2 = Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.DefaultV1Flags),
Values: uint64(0), // all flags in DefaultV1Flags are false
},
}
// DefaultOptionsV1 is the set of all options that define default v1 behavior.
var DefaultOptionsV1 = Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.DefaultV1Flags),
Values: uint64(jsonflags.DefaultV1Flags), // all flags in DefaultV1Flags are true
},
}
func (*Struct) JSONOptions(internal.NotForPublicUse) {}
// GetUnknownOption is injected by the "json" package to handle Options
// declared in that package so that "jsonopts" can handle them.
var GetUnknownOption = func(Struct, Options) (any, bool) { panic("unknown option") }
func GetOption[T any](opts Options, setter func(T) Options) (T, bool) {
// Collapse the options to *Struct to simplify lookup.
structOpts, ok := opts.(*Struct)
if !ok {
var structOpts2 Struct
structOpts2.Join(opts)
structOpts = &structOpts2
}
// Lookup the option based on the return value of the setter.
var zero T
switch opt := setter(zero).(type) {
case jsonflags.Bools:
v := structOpts.Flags.Get(opt)
ok := structOpts.Flags.Has(opt)
if !ok && opt == jsonflags.StringifyNumbers && structOpts.Flags.Get(jsonflags.StringTag) {
return any(true).(T), true // check also whether the option is specified via a `string` tag
}
return any(v).(T), ok
case experimentalFormatTagSupporter:
v := structOpts.Flags.Get(jsonflags.FormatTagSupported)
ok := structOpts.Flags.Has(jsonflags.FormatTagSupported)
return any(v).(T), ok
case Indent:
if !structOpts.Flags.Has(jsonflags.Indent) {
return zero, false
}
return any(structOpts.Indent).(T), true
case IndentPrefix:
if !structOpts.Flags.Has(jsonflags.IndentPrefix) {
return zero, false
}
return any(structOpts.IndentPrefix).(T), true
case ByteLimit:
if !structOpts.Flags.Has(jsonflags.ByteLimit) {
return zero, false
}
return any(structOpts.ByteLimit).(T), true
case DepthLimit:
if !structOpts.Flags.Has(jsonflags.DepthLimit) {
return zero, false
}
return any(structOpts.DepthLimit).(T), true
default:
v, ok := GetUnknownOption(*structOpts, opt)
return v.(T), ok
}
}
// JoinUnknownOption is injected by the "json" package to handle Options
// declared in that package so that "jsonopts" can handle them.
var JoinUnknownOption = func(Struct, Options) Struct { panic("unknown option") }
func (dst *Struct) Join(srcs ...Options) {
for _, src := range srcs {
switch src := src.(type) {
case nil:
continue
case jsonflags.Bools:
dst.Flags.Set(src)
case experimentalFormatTagSupporter:
if src.ExperimentalSupportFormatTag() {
dst.Flags.Set(jsonflags.FormatTagSupported | 1)
} else {
dst.Flags.Set(jsonflags.FormatTagSupported | 0)
}
case Indent:
dst.Flags.Set(jsonflags.Multiline | jsonflags.Indent | 1)
dst.Indent = string(src)
case IndentPrefix:
dst.Flags.Set(jsonflags.Multiline | jsonflags.IndentPrefix | 1)
dst.IndentPrefix = string(src)
case ByteLimit:
dst.Flags.Set(jsonflags.ByteLimit | 1)
dst.ByteLimit = int64(src)
case DepthLimit:
dst.Flags.Set(jsonflags.DepthLimit | 1)
dst.DepthLimit = int(src)
case *Struct:
dst.Flags.Join(src.Flags)
if src.Flags.Has(jsonflags.NonBooleanFlags) {
if src.Flags.Has(jsonflags.Indent) {
dst.Indent = src.Indent
}
if src.Flags.Has(jsonflags.IndentPrefix) {
dst.IndentPrefix = src.IndentPrefix
}
if src.Flags.Has(jsonflags.ByteLimit) {
dst.ByteLimit = src.ByteLimit
}
if src.Flags.Has(jsonflags.DepthLimit) {
dst.DepthLimit = src.DepthLimit
}
if src.Flags.Has(jsonflags.Marshalers) {
dst.Marshalers = src.Marshalers
}
if src.Flags.Has(jsonflags.Unmarshalers) {
dst.Unmarshalers = src.Unmarshalers
}
if src.Flags.Has(jsonflags.FormatTag) {
dst.Format = src.Format
}
}
default:
*dst = JoinUnknownOption(*dst, src)
}
}
}
// InitializeMultiline sets default options implied by Multiline.
func (s *Struct) InitializeMultiline() {
if !s.Flags.Has(jsonflags.SpaceAfterColon) {
s.Flags.Set(jsonflags.SpaceAfterColon | 1)
}
if !s.Flags.Has(jsonflags.SpaceAfterComma) {
s.Flags.Set(jsonflags.SpaceAfterComma | 0)
}
if !s.Flags.Has(jsonflags.Indent) {
s.Flags.Set(jsonflags.Indent | 1)
s.Indent = "\t"
}
}
// ChangedWhitespace reports whether whitespace values have changed.
func ChangedWhitespace(s1, s2 Struct) bool {
return s1.Flags.Get(jsonflags.Multiline) != s2.Flags.Get(jsonflags.Multiline) ||
s1.Flags.Get(jsonflags.SpaceAfterColon) != s2.Flags.Get(jsonflags.SpaceAfterColon) ||
s1.Flags.Get(jsonflags.SpaceAfterComma) != s2.Flags.Get(jsonflags.SpaceAfterComma) ||
(s2.Flags.Get(jsonflags.Multiline) && (s1.Indent != s2.Indent || s1.IndentPrefix != s2.IndentPrefix))
}
type (
Indent string // jsontext.WithIndent
IndentPrefix string // jsontext.WithIndentPrefix
ByteLimit int64 // jsontext.WithByteLimit
DepthLimit int // jsontext.WithDepthLimit
// type for jsonflags.Marshalers declared in "json" package
// type for jsonflags.Unmarshalers declared in "json" package
)
func (Indent) JSONOptions(internal.NotForPublicUse) {}
func (IndentPrefix) JSONOptions(internal.NotForPublicUse) {}
func (ByteLimit) JSONOptions(internal.NotForPublicUse) {}
func (DepthLimit) JSONOptions(internal.NotForPublicUse) {}

View File

@@ -0,0 +1,124 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsonopts
// NOTE: While [ExperimentalSupportFormatTag] is exported,
// it is in an internal package and thus inaccessible for public use.
//
// The [github.com/go-json-experiment/json] module is kept in sync
// with the Go standard standard library and will expose this option
// in a way that public code can now directly reference.
type supportFormatTag struct {
Options
v bool
}
var (
enableFormatTag = supportFormatTag{v: true}
disableFormatTag = supportFormatTag{v: false}
)
// ExperimentalSupportFormatTag is a marker method that is specially
// recognized by the "github.com/go-json-experiment/json" runtime.
func (o *supportFormatTag) ExperimentalSupportFormatTag() bool {
return o.v
}
// ExperimentalSupportFormatTag enables support for the `format` tag.
//
// WARNING: This is an experimental feature and will be removed in the future
// as either a failed experiment or be formally included in "github.com/go-json-experiment/json"
// in some semantically similar form, in which case, users of this option
// must migrate to the officially supported feature.
//
// The `format` tag was originally part of the "github.com/go-json-experiment/json" experiment
// but support for it was removed (see https://go.dev/issue/79071) for the
// initial release of "github.com/go-json-experiment/json" in light of the anticipation that
// the Go language might support typed struct tags (https://go.dev/issue/74472).
//
// Typed struct tags are a more expressive and type-safe way to express format
// attributes than the bespoke `format` tag option that implements a miniature
// domain-specific language (DSL) within the "json" package itself.
//
// While "github.com/go-json-experiment/json" was in the experimental phase,
// some users were already depending on the `format` tag option.
// This experimental option exists to provide a temporary workaround
// until the (hopeful) inclusion of typed struct tags and support for
// formatting directives in "github.com/go-json-experiment/json" using that language mechanism.
//
// This option enables support for the `format` tag option, which specifies
// a format flag used to specialize the formatting of the field value.
// The option is a key-value pair specified as "format:value" where
// the value must be either a literal consisting of letters and numbers
// (e.g., `format:RFC3339`) or a single-quoted string literal
// (e.g., `format:'2006-01-02'`). The interpretation of the format flag
// is determined by the struct field type.
//
// Go types with alternative representations are as follows:
//
// - A Go []byte or [N]byte is usually represented as a JSON string
// containing the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the bytes value is represented as a JSON array
// where each element recursively uses the JSON representation of each byte.
//
// - A Go float is usually represented as a JSON number.
// If the format is "nonfinite", then NaN, +Inf, and -Inf are represented as
// the JSON strings "NaN", "Infinity", and "-Infinity", respectively.
// Without the use of this format, such string values result in a [SemanticError].
//
// - A nil Go map is usually encoded using an empty JSON object.
// If the format is "emitnull", then a nil map is encoded as a JSON null.
// If the format is "emitempty", then a nil map is encoded as an empty JSON object,
// regardless of whether [FormatNilMapAsNull] is specified.
//
// - A nil Go slice is usually encoded using an empty JSON array.
// If the format is "emitnull", then a nil slice is encoded as a JSON null.
// If the format is "emitempty", then a nil slice is encoded as an empty JSON array,
// regardless of whether [FormatNilSliceAsNull] is specified.
//
// - A Go pointer usually uses the JSON representation of the underlying value.
// The format is forwarded to the marshaling and unmarshaling of the underlying type.
//
// - A Go [time.Time] is usually represented as a JSON string containing
// the timestamp formatted in RFC 3339 with nanosecond precision.
// If the format matches one of the format constants declared
// in the time package (e.g., RFC1123), then that format is used.
// If the format is "unix", "unixmilli", "unixmicro", or "unixnano",
// then the timestamp is represented as a possibly fractional JSON number
// of the number of seconds (or milliseconds, microseconds, or nanoseconds)
// since the Unix epoch, which is January 1st, 1970 at 00:00:00 UTC.
// To avoid a fractional component when encoding,
// round the timestamp to the relevant unit.
// Otherwise if non-empty, the format is used as-is and
// encoded using [time.Time.Format] and
// decoded using [time.Time.Parse].
//
// - A Go [time.Duration] usually has no default representation.
// If the format is "sec", "milli", "micro", or "nano",
// then the duration is represented as a possibly fractional JSON number
// of the number of seconds (or milliseconds, microseconds, or nanoseconds).
// To avoid a fractional component when encoding,
// round the duration to the relevant unit.
// If the format is "units", it is represented as a JSON string
// encoded using [time.Duration.String] and decoded using [time.ParseDuration]
// (e.g., "1h30m" for 1 hour 30 minutes).
// If the format is "iso8601", it is represented as a JSON string using the
// ISO 8601 standard for durations (e.g., "PT1H30M" for 1 hour 30 minutes)
// using only accurate units of hours, minutes, and seconds.
func ExperimentalSupportFormatTag(v bool) Options {
if v {
return &enableFormatTag
} else {
return &disableFormatTag
}
}

View File

@@ -0,0 +1,606 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsonwire
import (
"io"
"math"
"slices"
"strconv"
"unicode/utf16"
"unicode/utf8"
)
type ValueFlags uint
const (
_ ValueFlags = (1 << iota) / 2 // powers of two starting with zero
stringNonVerbatim // string cannot be naively treated as valid UTF-8
stringNonCanonical // string not formatted according to RFC 8785, section 3.2.2.2.
// TODO: Track whether a number is a non-integer?
)
func (f *ValueFlags) Join(f2 ValueFlags) { *f |= f2 }
func (f ValueFlags) IsVerbatim() bool { return f&stringNonVerbatim == 0 }
func (f ValueFlags) IsCanonical() bool { return f&stringNonCanonical == 0 }
// ConsumeWhitespace consumes leading JSON whitespace per RFC 7159, section 2.
func ConsumeWhitespace(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
for len(b) > n && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') {
n++
}
return n
}
// ConsumeNull consumes the next JSON null literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeNull(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "null"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeFalse consumes the next JSON false literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeFalse(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "false"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeTrue consumes the next JSON true literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeTrue(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "true"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeLiteral consumes the next JSON literal per RFC 7159, section 3.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
func ConsumeLiteral(b []byte, lit string) (n int, err error) {
for i := 0; i < len(b) && i < len(lit); i++ {
if b[i] != lit[i] {
return i, NewInvalidCharacterError(b[i:], "in literal "+lit+" (expecting "+strconv.QuoteRune(rune(lit[i]))+")")
}
}
if len(b) < len(lit) {
return len(b), io.ErrUnexpectedEOF
}
return len(lit), nil
}
// ConsumeSimpleString consumes the next JSON string per RFC 7159, section 7
// but is limited to the grammar for an ASCII string without escape sequences.
// It returns 0 if it is invalid or more complicated than a simple string,
// in which case [ConsumeString] should be called.
//
// It rejects '<', '>', and '&' for compatibility reasons since these were
// always escaped in the v1 implementation. Thus, if this function reports
// non-zero then we know that the string would be encoded the same way
// under both v1 or v2 escape semantics.
func ConsumeSimpleString(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[0] == '"' {
n++
for len(b) > n && b[n] < utf8.RuneSelf && escapeASCII[b[n]] == 0 {
n++
}
if uint(len(b)) > uint(n) && b[n] == '"' {
n++
return n
}
}
return 0
}
// ConsumeString consumes the next JSON string per RFC 7159, section 7.
// If validateUTF8 is false, then this allows the presence of invalid UTF-8
// characters within the string itself.
// It reports the number of bytes consumed and whether an error was encountered.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
func ConsumeString(flags *ValueFlags, b []byte, validateUTF8 bool) (n int, err error) {
return ConsumeStringResumable(flags, b, 0, validateUTF8)
}
// ConsumeStringResumable is identical to [ConsumeString] but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeStringResumable(flags *ValueFlags, b []byte, resumeOffset int, validateUTF8 bool) (n int, err error) {
// Consume the leading double quote.
switch {
case resumeOffset > 0:
n = resumeOffset // already handled the leading quote
case uint(len(b)) == 0:
return n, io.ErrUnexpectedEOF
case b[0] == '"':
n++
default:
return n, NewInvalidCharacterError(b[n:], `at start of string (expecting '"')`)
}
// Consume every character in the string.
for uint(len(b)) > uint(n) {
// Optimize for long sequences of unescaped characters.
noEscape := func(c byte) bool {
return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"'
}
for uint(len(b)) > uint(n) && noEscape(b[n]) {
n++
}
if uint(len(b)) <= uint(n) {
return n, io.ErrUnexpectedEOF
}
// Check for terminating double quote.
if b[n] == '"' {
n++
return n, nil
}
switch r, rn := utf8.DecodeRune(b[n:]); {
// Handle UTF-8 encoded byte sequence.
// Due to specialized handling of ASCII above, we know that
// all normal sequences at this point must be 2 bytes or larger.
case rn > 1:
n += rn
// Handle escape sequence.
case r == '\\':
flags.Join(stringNonVerbatim)
resumeOffset = n
if uint(len(b)) < uint(n+2) {
return resumeOffset, io.ErrUnexpectedEOF
}
switch r := b[n+1]; r {
case '/':
// Forward slash is the only character with 3 representations.
// Per RFC 8785, section 3.2.2.2., this must not be escaped.
flags.Join(stringNonCanonical)
n += 2
case '"', '\\', 'b', 'f', 'n', 'r', 't':
n += 2
case 'u':
if uint(len(b)) < uint(n+6) {
if hasEscapedUTF16Prefix(b[n:], false) {
return resumeOffset, io.ErrUnexpectedEOF
}
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n:])
}
v1, ok := parseHexUint16(b[n+2 : n+6])
if !ok {
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n : n+6])
}
// Only certain control characters can use the \uFFFF notation
// for canonical formatting (per RFC 8785, section 3.2.2.2.).
switch v1 {
// \uFFFF notation not permitted for these characters.
case '\b', '\f', '\n', '\r', '\t':
flags.Join(stringNonCanonical)
default:
// \uFFFF notation only permitted for control characters.
if v1 >= ' ' {
flags.Join(stringNonCanonical)
} else {
// \uFFFF notation must be lower case.
for _, c := range b[n+2 : n+6] {
if 'A' <= c && c <= 'F' {
flags.Join(stringNonCanonical)
}
}
}
}
n += 6
r := rune(v1)
if validateUTF8 && utf16.IsSurrogate(r) {
if uint(len(b)) < uint(n+6) {
if hasEscapedUTF16Prefix(b[n:], true) {
return resumeOffset, io.ErrUnexpectedEOF
}
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6:])
} else if v2, ok := parseHexUint16(b[n+2 : n+6]); b[n] != '\\' || b[n+1] != 'u' || !ok {
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6])
} else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError {
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6])
} else {
n += 6
}
}
default:
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n : n+2])
}
// Handle invalid UTF-8.
case r == utf8.RuneError:
if !utf8.FullRune(b[n:]) {
return n, io.ErrUnexpectedEOF
}
flags.Join(stringNonVerbatim | stringNonCanonical)
if validateUTF8 {
return n, ErrInvalidUTF8
}
n++
// Handle invalid control characters.
case r < ' ':
flags.Join(stringNonVerbatim | stringNonCanonical)
return n, NewInvalidCharacterError(b[n:], "in string (expecting non-control character)")
default:
panic("BUG: unhandled character " + QuoteRune(b[n:]))
}
}
return n, io.ErrUnexpectedEOF
}
// AppendUnquote appends the unescaped form of a JSON string in src to dst.
// Any invalid UTF-8 within the string will be replaced with utf8.RuneError,
// but the error will be specified as having encountered such an error.
// The input must be an entire JSON string with no surrounding whitespace.
func AppendUnquote(dst, src []byte) (v []byte, err error) {
dst = slices.Grow(dst, len(src))
// Consume the leading double quote.
var i, n int
switch {
case uint(len(src)) == 0:
return dst, io.ErrUnexpectedEOF
case src[0] == '"':
i, n = 1, 1
default:
return dst, NewInvalidCharacterError(src, `at start of string (expecting '"')`)
}
// Consume every character in the string.
for uint(len(src)) > uint(n) {
// Optimize for long sequences of unescaped characters.
noEscape := func(c byte) bool {
return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"'
}
for uint(len(src)) > uint(n) && noEscape(src[n]) {
n++
}
if uint(len(src)) <= uint(n) {
dst = append(dst, src[i:n]...)
return dst, io.ErrUnexpectedEOF
}
// Check for terminating double quote.
if src[n] == '"' {
dst = append(dst, src[i:n]...)
n++
if n < len(src) {
err = NewInvalidCharacterError(src[n:], "after string value")
}
return dst, err
}
switch r, rn := utf8.DecodeRune(src[n:]); {
// Handle UTF-8 encoded byte sequence.
// Due to specialized handling of ASCII above, we know that
// all normal sequences at this point must be 2 bytes or larger.
case rn > 1:
n += rn
// Handle escape sequence.
case r == '\\':
dst = append(dst, src[i:n]...)
// Handle escape sequence.
if uint(len(src)) < uint(n+2) {
return dst, io.ErrUnexpectedEOF
}
switch r := src[n+1]; r {
case '"', '\\', '/':
dst = append(dst, r)
n += 2
case 'b':
dst = append(dst, '\b')
n += 2
case 'f':
dst = append(dst, '\f')
n += 2
case 'n':
dst = append(dst, '\n')
n += 2
case 'r':
dst = append(dst, '\r')
n += 2
case 't':
dst = append(dst, '\t')
n += 2
case 'u':
if uint(len(src)) < uint(n+6) {
if hasEscapedUTF16Prefix(src[n:], false) {
return dst, io.ErrUnexpectedEOF
}
return dst, NewInvalidEscapeSequenceError(src[n:])
}
v1, ok := parseHexUint16(src[n+2 : n+6])
if !ok {
return dst, NewInvalidEscapeSequenceError(src[n : n+6])
}
n += 6
// Check whether this is a surrogate half.
r := rune(v1)
if utf16.IsSurrogate(r) {
r = utf8.RuneError // assume failure unless the following succeeds
if uint(len(src)) < uint(n+6) {
if hasEscapedUTF16Prefix(src[n:], true) {
return utf8.AppendRune(dst, r), io.ErrUnexpectedEOF
}
err = NewInvalidEscapeSequenceError(src[n-6:])
} else if v2, ok := parseHexUint16(src[n+2 : n+6]); src[n] != '\\' || src[n+1] != 'u' || !ok {
err = NewInvalidEscapeSequenceError(src[n-6 : n+6])
} else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError {
err = NewInvalidEscapeSequenceError(src[n-6 : n+6])
} else {
n += 6
}
}
dst = utf8.AppendRune(dst, r)
default:
return dst, NewInvalidEscapeSequenceError(src[n : n+2])
}
i = n
// Handle invalid UTF-8.
case r == utf8.RuneError:
dst = append(dst, src[i:n]...)
if !utf8.FullRune(src[n:]) {
return dst, io.ErrUnexpectedEOF
}
// NOTE: An unescaped string may be longer than the escaped string
// because invalid UTF-8 bytes are being replaced.
dst = append(dst, "\uFFFD"...)
n += rn
i = n
err = ErrInvalidUTF8
// Handle invalid control characters.
case r < ' ':
dst = append(dst, src[i:n]...)
return dst, NewInvalidCharacterError(src[n:], "in string (expecting non-control character)")
default:
panic("BUG: unhandled character " + QuoteRune(src[n:]))
}
}
dst = append(dst, src[i:n]...)
return dst, io.ErrUnexpectedEOF
}
// hasEscapedUTF16Prefix reports whether b is possibly
// the truncated prefix of a \uFFFF escape sequence.
func hasEscapedUTF16Prefix(b []byte, lowerSurrogateHalf bool) bool {
for i := range len(b) {
switch c := b[i]; {
case i == 0 && c != '\\':
return false
case i == 1 && c != 'u':
return false
case i == 2 && lowerSurrogateHalf && c != 'd' && c != 'D':
return false // not within ['\uDC00':'\uDFFF']
case i == 3 && lowerSurrogateHalf && !('c' <= c && c <= 'f') && !('C' <= c && c <= 'F'):
return false // not within ['\uDC00':'\uDFFF']
case i >= 2 && i < 6 && !('0' <= c && c <= '9') && !('a' <= c && c <= 'f') && !('A' <= c && c <= 'F'):
return false
}
}
return true
}
// UnquoteMayCopy returns the unescaped form of b.
// If there are no escaped characters, the output is simply a subslice of
// the input with the surrounding quotes removed.
// Otherwise, a new buffer is allocated for the output.
// It assumes the input is valid.
func UnquoteMayCopy(b []byte, isVerbatim bool) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if isVerbatim {
return b[len(`"`) : len(b)-len(`"`)]
}
b, _ = AppendUnquote(nil, b)
return b
}
// ConsumeSimpleNumber consumes the next JSON number per RFC 7159, section 6
// but is limited to the grammar for a non-negative integer.
// It returns 0 if it is invalid or more complicated than a simple integer,
// in which case ConsumeNumber should be called.
func ConsumeSimpleNumber(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 {
if b[0] == '0' {
n++
} else if '1' <= b[0] && b[0] <= '9' {
n++
for len(b) > n && ('0' <= b[n] && b[n] <= '9') {
n++
}
} else {
return 0
}
if uint(len(b)) <= uint(n) || (b[n] != '.' && b[n] != 'e' && b[n] != 'E') {
return n
}
}
return 0
}
type ConsumeNumberState uint
const (
consumeNumberInit ConsumeNumberState = iota
beforeIntegerDigits
withinIntegerDigits
beforeFractionalDigits
withinFractionalDigits
beforeExponentDigits
withinExponentDigits
)
// ConsumeNumber consumes the next JSON number per RFC 7159, section 6.
// It reports the number of bytes consumed and whether an error was encountered.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
//
// Note that JSON numbers are not self-terminating.
// If the entire input is consumed, then the caller needs to consider whether
// there may be subsequent unread data that may still be part of this number.
func ConsumeNumber(b []byte) (n int, err error) {
n, _, err = ConsumeNumberResumable(b, 0, consumeNumberInit)
return n, err
}
// ConsumeNumberResumable is identical to ConsumeNumber but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeNumberResumable(b []byte, resumeOffset int, state ConsumeNumberState) (n int, _ ConsumeNumberState, err error) {
// Jump to the right state when resuming from a partial consumption.
n = resumeOffset
if state > consumeNumberInit {
switch state {
case withinIntegerDigits, withinFractionalDigits, withinExponentDigits:
// Consume leading digits.
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
if uint(len(b)) <= uint(n) {
return n, state, nil // still within the same state
}
state++ // switches "withinX" to "beforeY" where Y is the state after X
}
switch state {
case beforeIntegerDigits:
goto beforeInteger
case beforeFractionalDigits:
goto beforeFractional
case beforeExponentDigits:
goto beforeExponent
default:
return n, state, nil
}
}
// Consume required integer component (with optional minus sign).
beforeInteger:
resumeOffset = n
if uint(len(b)) > 0 && b[0] == '-' {
n++
}
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeIntegerDigits, io.ErrUnexpectedEOF
case b[n] == '0':
n++
state = beforeFractionalDigits
case '1' <= b[n] && b[n] <= '9':
n++
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinIntegerDigits
default:
return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)")
}
// Consume optional fractional component.
beforeFractional:
if uint(len(b)) > uint(n) && b[n] == '.' {
resumeOffset = n
n++
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeFractionalDigits, io.ErrUnexpectedEOF
case '0' <= b[n] && b[n] <= '9':
n++
default:
return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)")
}
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinFractionalDigits
}
// Consume optional exponent component.
beforeExponent:
if uint(len(b)) > uint(n) && (b[n] == 'e' || b[n] == 'E') {
resumeOffset = n
n++
if uint(len(b)) > uint(n) && (b[n] == '-' || b[n] == '+') {
n++
}
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeExponentDigits, io.ErrUnexpectedEOF
case '0' <= b[n] && b[n] <= '9':
n++
default:
return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)")
}
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinExponentDigits
}
return n, state, nil
}
// parseHexUint16 is similar to strconv.ParseUint,
// but operates directly on []byte and is optimized for base-16.
// See https://go.dev/issue/42429.
func parseHexUint16(b []byte) (v uint16, ok bool) {
if len(b) != 4 {
return 0, false
}
for i := range 4 {
c := b[i]
switch {
case '0' <= c && c <= '9':
c = c - '0'
case 'a' <= c && c <= 'f':
c = 10 + c - 'a'
case 'A' <= c && c <= 'F':
c = 10 + c - 'A'
default:
return 0, false
}
v = v*16 + uint16(c)
}
return v, true
}
// ParseUint parses b as a decimal unsigned integer according to
// a strict subset of the JSON number grammar, returning the value if valid.
// It returns (0, false) if there is a syntax error and
// returns (math.MaxUint64, false) if there is an overflow.
func ParseUint(b []byte) (v uint64, ok bool) {
const unsafeWidth = 20 // len(fmt.Sprint(uint64(math.MaxUint64)))
var n int
for ; len(b) > n && ('0' <= b[n] && b[n] <= '9'); n++ {
v = 10*v + uint64(b[n]-'0')
}
switch {
case n == 0 || len(b) != n || (b[0] == '0' && string(b) != "0"):
return 0, false
case n >= unsafeWidth && (b[0] != '1' || v < 1e19 || n > unsafeWidth):
return math.MaxUint64, false
}
return v, true
}

View File

@@ -0,0 +1,290 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsonwire
import (
"math"
"slices"
"strconv"
"unicode/utf16"
"unicode/utf8"
"github.com/go-json-experiment/json/internal/jsonflags"
)
// escapeASCII reports whether the ASCII character needs to be escaped.
// It conservatively assumes EscapeForHTML.
var escapeASCII = [...]uint8{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters
0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // escape '"' and '&'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, // escape '<' and '>'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // escape '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
}
// NeedEscape reports whether src needs escaping of any characters.
// It conservatively assumes EscapeForHTML and EscapeForJS.
// It reports true for inputs with invalid UTF-8.
func NeedEscape(src []byte) bool {
var i int
for uint(len(src)) > uint(i) {
if c := src[i]; c < utf8.RuneSelf {
if escapeASCII[c] > 0 {
return true
}
i++
} else {
r, rn := utf8.DecodeRune(src[i:])
if r == utf8.RuneError || r == '\u2028' || r == '\u2029' {
return true
}
i += rn
}
}
return false
}
// AppendQuote appends src to dst as a JSON string per RFC 7159, section 7.
//
// It takes in flags and respects the following:
// - EscapeForHTML escapes '<', '>', and '&'.
// - EscapeForJS escapes '\u2028' and '\u2029'.
// - AllowInvalidUTF8 avoids reporting an error for invalid UTF-8.
//
// Regardless of whether AllowInvalidUTF8 is specified,
// invalid bytes are replaced with the Unicode replacement character ('\ufffd').
// If no escape flags are set, then the shortest representable form is used,
// which is also the canonical form for strings (RFC 8785, section 3.2.2.2).
func AppendQuote(dst, src []byte, flags *jsonflags.Flags) ([]byte, error) {
var i, n int
var hasInvalidUTF8 bool
dst = slices.Grow(dst, len(`"`)+len(src)+len(`"`))
dst = append(dst, '"')
for uint(len(src)) > uint(n) {
if c := src[n]; c < utf8.RuneSelf {
// Handle single-byte ASCII.
n++
if escapeASCII[c] == 0 {
continue // no escaping possibly needed
}
// Handle escaping of single-byte ASCII.
if !(c == '<' || c == '>' || c == '&') || flags.Get(jsonflags.EscapeForHTML) {
dst = append(dst, src[i:n-1]...)
dst = appendEscapedASCII(dst, c)
i = n
}
} else {
// Handle multi-byte Unicode.
r, rn := utf8.DecodeRune(src[n:])
n += rn
if r != utf8.RuneError && r != '\u2028' && r != '\u2029' {
continue // no escaping possibly needed
}
// Handle escaping of multi-byte Unicode.
switch {
case isInvalidUTF8(r, rn):
hasInvalidUTF8 = true
dst = append(dst, src[i:n-rn]...)
dst = append(dst, "\ufffd"...)
i = n
case (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS):
dst = append(dst, src[i:n-rn]...)
dst = appendEscapedUnicode(dst, r)
i = n
}
}
}
dst = append(dst, src[i:n]...)
dst = append(dst, '"')
if hasInvalidUTF8 && !flags.Get(jsonflags.AllowInvalidUTF8) {
return dst, ErrInvalidUTF8
}
return dst, nil
}
func appendEscapedASCII(dst []byte, c byte) []byte {
switch c {
case '"', '\\':
dst = append(dst, '\\', c)
case '\b':
dst = append(dst, "\\b"...)
case '\f':
dst = append(dst, "\\f"...)
case '\n':
dst = append(dst, "\\n"...)
case '\r':
dst = append(dst, "\\r"...)
case '\t':
dst = append(dst, "\\t"...)
default:
dst = appendEscapedUTF16(dst, uint16(c))
}
return dst
}
func appendEscapedUnicode(dst []byte, r rune) []byte {
if r1, r2 := utf16.EncodeRune(r); r1 != '\ufffd' && r2 != '\ufffd' {
dst = appendEscapedUTF16(dst, uint16(r1))
dst = appendEscapedUTF16(dst, uint16(r2))
} else {
dst = appendEscapedUTF16(dst, uint16(r))
}
return dst
}
func appendEscapedUTF16(dst []byte, x uint16) []byte {
const hex = "0123456789abcdef"
return append(dst, '\\', 'u', hex[(x>>12)&0xf], hex[(x>>8)&0xf], hex[(x>>4)&0xf], hex[(x>>0)&0xf])
}
// ReformatString consumes a JSON string from src and appends it to dst,
// reformatting it if necessary according to the specified flags.
// It returns the appended output and the number of consumed input bytes.
func ReformatString(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) {
// TODO: Should this update ValueFlags as input?
var valFlags ValueFlags
n, err := ConsumeString(&valFlags, src, !flags.Get(jsonflags.AllowInvalidUTF8))
if err != nil {
return dst, n, err
}
// If the output requires no special escapes, and the input
// is already in canonical form or should be preserved verbatim,
// then directly copy the input to the output.
if !flags.Get(jsonflags.AnyEscape) &&
(valFlags.IsCanonical() || flags.Get(jsonflags.PreserveRawStrings)) {
dst = append(dst, src[:n]...) // copy the string verbatim
return dst, n, nil
}
// Under [jsonflags.PreserveRawStrings], any pre-escaped sequences
// remain escaped, however we still need to respect the
// [jsonflags.EscapeForHTML] and [jsonflags.EscapeForJS] options.
if flags.Get(jsonflags.PreserveRawStrings) {
var i, lastAppendIndex int
for i < n {
if c := src[i]; c < utf8.RuneSelf {
if (c == '<' || c == '>' || c == '&') && flags.Get(jsonflags.EscapeForHTML) {
dst = append(dst, src[lastAppendIndex:i]...)
dst = appendEscapedASCII(dst, c)
lastAppendIndex = i + 1
}
i++
} else {
r, rn := utf8.DecodeRune(src[i:])
if (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS) {
dst = append(dst, src[lastAppendIndex:i]...)
dst = appendEscapedUnicode(dst, r)
lastAppendIndex = i + rn
}
i += rn
}
}
return append(dst, src[lastAppendIndex:n]...), n, nil
}
// The input contains characters that might need escaping,
// unnecessary escape sequences, or invalid UTF-8.
// Perform a round-trip unquote and quote to properly reformat
// these sequences according to the current flags.
b, _ := AppendUnquote(nil, src[:n])
dst, _ = AppendQuote(dst, b, flags)
return dst, n, nil
}
// AppendFloat appends src to dst as a JSON number per RFC 7159, section 6.
// It formats numbers similar to the ES6 number-to-string conversion.
// See https://go.dev/issue/14135.
//
// The output is identical to ECMA-262, 6th edition, section 7.1.12.1 and with
// RFC 8785, section 3.2.2.3 for 64-bit floating-point numbers except for -0,
// which is formatted as -0 instead of just 0.
//
// For 32-bit floating-point numbers,
// the output is a 32-bit equivalent of the algorithm.
// Note that ECMA-262 specifies no algorithm for 32-bit numbers.
func AppendFloat(dst []byte, src float64, bits int) []byte {
if bits == 32 {
src = float64(float32(src))
}
abs := math.Abs(src)
fmt := byte('f')
if abs != 0 {
if bits == 64 && (float64(abs) < 1e-6 || float64(abs) >= 1e21) ||
bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
fmt = 'e'
}
}
dst = strconv.AppendFloat(dst, src, fmt, -1, bits)
if fmt == 'e' {
// Clean up e-09 to e-9.
n := len(dst)
if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' {
dst[n-2] = dst[n-1]
dst = dst[:n-1]
}
}
return dst
}
// ReformatNumber consumes a JSON number from src and appends it to dst,
// canonicalizing it if specified.
// It returns the appended output and the number of consumed input bytes.
func ReformatNumber(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) {
n, err := ConsumeNumber(src)
if err != nil {
return dst, n, err
}
if !flags.Get(jsonflags.CanonicalizeNumbers) {
dst = append(dst, src[:n]...) // copy the number verbatim
return dst, n, nil
}
// Identify the kind of number.
var isFloat bool
for _, c := range src[:n] {
if c == '.' || c == 'e' || c == 'E' {
isFloat = true // has fraction or exponent
break
}
}
// Check if need to canonicalize this kind of number.
switch {
case string(src[:n]) == "-0":
break // canonicalize -0 as 0 regardless of kind
case isFloat:
if !flags.Get(jsonflags.CanonicalizeRawFloats) {
dst = append(dst, src[:n]...) // copy the number verbatim
return dst, n, nil
}
default:
// As an optimization, we can copy integer numbers below 2⁵³ verbatim
// since the canonical form is always identical.
const maxExactIntegerDigits = 16 // len(strconv.AppendUint(nil, 1<<53, 10))
if !flags.Get(jsonflags.CanonicalizeRawInts) || n < maxExactIntegerDigits {
dst = append(dst, src[:n]...) // copy the number verbatim
return dst, n, nil
}
}
// Parse and reformat the number (which uses a canonical format).
fv, _ := strconv.ParseFloat(string(src[:n]), 64)
switch {
case fv == 0:
fv = 0 // normalize negative zero as just zero
case math.IsInf(fv, +1):
fv = +math.MaxFloat64
case math.IsInf(fv, -1):
fv = -math.MaxFloat64
}
return AppendFloat(dst, fv, 64), n, nil
}

View File

@@ -0,0 +1,209 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
// Package jsonwire implements stateless functionality for handling JSON text.
package jsonwire
import (
"cmp"
"errors"
"strconv"
"strings"
"unicode"
"unicode/utf16"
"unicode/utf8"
)
// TrimSuffixWhitespace trims JSON whitespace from the end of b.
func TrimSuffixWhitespace(b []byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
n := len(b) - 1
for n >= 0 && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') {
n--
}
return b[:n+1]
}
// TrimSuffixString trims a valid JSON string at the end of b.
// The behavior is undefined if there is not a valid JSON string present.
func TrimSuffixString(b []byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[len(b)-1] == '"' {
b = b[:len(b)-1]
}
for len(b) >= 2 && !(b[len(b)-1] == '"' && b[len(b)-2] != '\\') {
b = b[:len(b)-1] // trim all characters except an unescaped quote
}
if len(b) > 0 && b[len(b)-1] == '"' {
b = b[:len(b)-1]
}
return b
}
// HasSuffixByte reports whether b ends with c.
func HasSuffixByte(b []byte, c byte) bool {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
return len(b) > 0 && b[len(b)-1] == c
}
// TrimSuffixByte removes c from the end of b if it is present.
func TrimSuffixByte(b []byte, c byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[len(b)-1] == c {
return b[:len(b)-1]
}
return b
}
// QuoteRune quotes the first rune in the input.
func QuoteRune(b []byte) string {
r, n := utf8.DecodeRune(b)
if r == utf8.RuneError && n == 1 {
return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'`
}
return strconv.QuoteRune(r)
}
// CompareUTF16 lexicographically compares x to y according
// to the UTF-16 codepoints of the UTF-8 encoded input strings.
// This implements the ordering specified in RFC 8785, section 3.2.3.
func CompareUTF16(x, y []byte) int {
// NOTE: This is an optimized, mostly allocation-free implementation
// of CompareUTF16Simple in wire_test.go. FuzzCompareUTF16 verifies that the
// two implementations agree on the result of comparing any two strings.
isUTF16Self := func(r rune) bool {
return ('\u0000' <= r && r <= '\uD7FF') || ('\uE000' <= r && r <= '\uFFFF')
}
for {
if len(x) == 0 || len(y) == 0 {
return cmp.Compare(len(x), len(y))
}
// ASCII fast-path.
if x[0] < utf8.RuneSelf || y[0] < utf8.RuneSelf {
if x[0] != y[0] {
return cmp.Compare(x[0], y[0])
}
x, y = x[1:], y[1:]
continue
}
// Decode next pair of runes as UTF-8.
rx, nx := utf8.DecodeRune(x)
ry, ny := utf8.DecodeRune(y)
selfx := isUTF16Self(rx)
selfy := isUTF16Self(ry)
switch {
// The x rune is a single UTF-16 codepoint, while
// the y rune is a surrogate pair of UTF-16 codepoints.
case selfx && !selfy:
ry, _ = utf16.EncodeRune(ry)
// The y rune is a single UTF-16 codepoint, while
// the x rune is a surrogate pair of UTF-16 codepoints.
case selfy && !selfx:
rx, _ = utf16.EncodeRune(rx)
}
if rx != ry {
return cmp.Compare(rx, ry)
}
// Check for invalid UTF-8, in which case,
// we just perform a byte-for-byte comparison.
if isInvalidUTF8(rx, nx) || isInvalidUTF8(ry, ny) {
if x[0] != y[0] {
return cmp.Compare(x[0], y[0])
}
}
x, y = x[nx:], y[ny:]
}
}
// TODO(https://go.dev/issue/70547): Use utf8.ErrInvalid instead.
var ErrInvalidUTF8 = errors.New("invalid UTF-8")
func NewInvalidCharacterError[Bytes ~[]byte | ~string](prefix Bytes, where string) error {
_, n := utf8.DecodeRune([]byte(prefix))
return &InvalidTextError{"character", string(prefix[:n]), where}
}
func NewInvalidEscapeSequenceError[Bytes ~[]byte | ~string](what Bytes) error {
if len(what) > 6 {
return &InvalidTextError{"surrogate pair", string(what), "in string"}
}
return &InvalidTextError{"escape sequence", string(what), "in string"}
}
type InvalidTextError struct {
Label string // e.g., "character" | "escape sequence" | "surrogate pair"
What string // raw invalid text
Where string // e.g., "in string" | "at start of value"
}
func (e *InvalidTextError) Error() string {
var what string
needEscape := strings.ContainsFunc(e.What, func(r rune) bool {
return r == '`' || r == utf8.RuneError || unicode.IsSpace(r) || !unicode.IsPrint(r)
})
switch {
case utf8.RuneCount([]byte(e.What)) == 1:
what = QuoteRune([]byte(e.What))
case needEscape:
what = strconv.Quote(e.What)
default:
what = "`" + e.What + "`"
}
return strings.TrimSuffix("invalid "+e.Label+" "+what+" "+e.Where, " ")
}
// TruncatePointer optionally truncates the JSON pointer,
// enforcing that the length roughly does not exceed n.
func TruncatePointer(s string, n int) string {
if len(s) <= n {
return s
}
i := n / 2
j := len(s) - n/2
// Avoid truncating a name if there are multiple names present.
if k := strings.LastIndexByte(s[:i], '/'); k > 0 {
i = k
}
if k := strings.IndexByte(s[j:], '/'); k >= 0 {
j += k + len("/")
}
// Avoid truncation in the middle of a UTF-8 rune.
for i > 0 && isInvalidUTF8(utf8.DecodeLastRuneInString(s[:i])) {
i--
}
for j < len(s) && isInvalidUTF8(utf8.DecodeRuneInString(s[j:])) {
j++
}
// Determine the right middle fragment to use.
var middle string
switch strings.Count(s[i:j], "/") {
case 0:
middle = "…"
case 1:
middle = "…/…"
default:
middle = "…/…/…"
}
if strings.HasPrefix(s[i:j], "/") && middle != "…" {
middle = strings.TrimPrefix(middle, "…")
}
if strings.HasSuffix(s[i:j], "/") && middle != "…" {
middle = strings.TrimSuffix(middle, "…")
}
return s[:i] + middle + s[j:]
}
func isInvalidUTF8(r rune, rn int) bool {
return r == utf8.RuneError && rn == 1
}

View File

@@ -0,0 +1,555 @@
// Copyright 2025 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated by alias_gen.go; DO NOT EDIT.
//go:build goexperiment.jsonv2 && go1.25
// Package jsontext implements syntactic processing of JSON
// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
// The [Encoder] and [Decoder] types are used to encode or decode
// a stream of JSON tokens or values.
//
// # Tokens and Values
//
// A JSON token refers to the basic structural elements of JSON:
//
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a begin or end delimiter for a JSON object (i.e., '{' or '}')
// - a begin or end delimiter for a JSON array (i.e., '[' or ']')
//
// A JSON token is represented by the [Token] type in Go. Technically,
// there are two additional structural characters (i.e., ':' and ','),
// but there is no [Token] representation for them since their presence
// can be inferred by the structure of the JSON grammar itself.
// For example, there must always be an implicit colon between
// the name and value of a JSON object member.
//
// A JSON value refers to a complete unit of JSON data:
//
// - a JSON literal, string, or number
// - a JSON object (e.g., `{"name":"value"}`)
// - a JSON array (e.g., `[1,2,3]`)
//
// A JSON value is represented by the [Value] type in Go and is a []byte
// containing the raw textual representation of the value. There is some overlap
// between tokens and values as both contain literals, strings, and numbers.
// However, only a value can represent the entirety of a JSON object or array.
//
// The [Encoder] and [Decoder] types contain methods to read or write the next
// [Token] or [Value] in a sequence. They maintain a state machine to validate
// whether the sequence of JSON tokens and/or values produces a valid JSON.
// [Options] may be passed to the [NewEncoder] or [NewDecoder] constructors
// to configure the syntactic behavior of encoding and decoding.
//
// # Terminology
//
// The terms "encode" and "decode" are used for syntactic functionality
// that is concerned with processing JSON based on its grammar, and
// the terms "marshal" and "unmarshal" are used for semantic functionality
// that determines the meaning of JSON values as Go values and vice-versa.
// This package (i.e., [jsontext]) deals with JSON at a syntactic layer,
// while [encoding/json/v2] deals with JSON at a semantic layer.
// The goal is to provide a clear distinction between functionality that
// is purely concerned with encoding versus that of marshaling.
// For example, one can directly encode a stream of JSON tokens without
// needing to marshal a concrete Go value representing them.
// Similarly, one can decode a stream of JSON tokens without
// needing to unmarshal them into a concrete Go value.
//
// This package uses JSON terminology when discussing JSON, which may differ
// from related concepts in Go or elsewhere in computing literature.
//
// - a JSON "object" refers to an unordered collection of name/value members.
// - a JSON "array" refers to an ordered sequence of elements.
// - a JSON "value" refers to either a literal (i.e., null, false, or true),
// string, number, object, or array.
//
// See RFC 8259 for more information.
//
// # Specifications
//
// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259,
// and RFC 8785. Each RFC is generally a stricter subset of another RFC.
// In increasing order of strictness:
//
// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8
// and also do not require (but recommend) that object names be unique.
// - RFC 8259 requires the use of UTF-8,
// but does not require (but recommends) that object names be unique.
// - RFC 7493 requires the use of UTF-8
// and also requires that object names be unique.
// - RFC 8785 defines a canonical representation. It requires the use of UTF-8
// and also requires that object names be unique and in a specific ordering.
// It specifies exactly how strings and numbers must be formatted.
//
// The primary difference between RFC 4627 and RFC 7159 is that the former
// restricted top-level values to only JSON objects and arrays, while
// RFC 7159 and subsequent RFCs permit top-level values to additionally be
// JSON nulls, booleans, strings, or numbers.
//
// By default, this package operates on RFC 7493, but can be configured
// to operate according to the other RFC specifications.
// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it.
// In particular, it makes specific choices about behavior that RFC 8259
// leaves as undefined in order to ensure greater interoperability.
//
// # Security Considerations
//
// See the "Security Considerations" section in [encoding/json/v2].
package jsontext
import (
"encoding/json/jsontext"
"io"
)
// Decoder is a streaming decoder for raw JSON tokens and values.
// It is used to read a stream of top-level JSON values,
// each separated by optional whitespace characters.
//
// [Decoder.ReadToken] and [Decoder.ReadValue] calls may be interleaved.
// For example, the following JSON value:
//
// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}}
//
// can be parsed with the following calls (ignoring errors for brevity):
//
// d.ReadToken() // {
// d.ReadToken() // "name"
// d.ReadToken() // "value"
// d.ReadValue() // "array"
// d.ReadToken() // [
// d.ReadToken() // null
// d.ReadToken() // false
// d.ReadValue() // true
// d.ReadToken() // 3.14159
// d.ReadToken() // ]
// d.ReadValue() // "object"
// d.ReadValue() // {"k":"v"}
// d.ReadToken() // }
//
// The above is one of many possible sequences of calls and
// may not represent the most sensible method to call for any given token/value.
// For example, it is probably more common to call [Decoder.ReadToken] to obtain a
// string token for object names.
type Decoder = jsontext.Decoder
// NewDecoder constructs a new streaming decoder reading from r.
//
// If r is a [bytes.Buffer], then the decoder parses directly from the buffer
// without first copying the contents to an intermediate buffer.
// Additional writes to the buffer must not occur while the decoder is in use.
func NewDecoder(r io.Reader, opts ...Options) *Decoder {
return jsontext.NewDecoder(r, opts...)
}
// Encoder is a streaming encoder from raw JSON tokens and values.
// It is used to write a stream of top-level JSON values,
// each terminated with a newline character.
//
// [Encoder.WriteToken] and [Encoder.WriteValue] calls may be interleaved.
// For example, the following JSON value:
//
// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}}
//
// can be composed with the following calls (ignoring errors for brevity):
//
// e.WriteToken(BeginObject) // {
// e.WriteToken(String("name")) // "name"
// e.WriteToken(String("value")) // "value"
// e.WriteValue(Value(`"array"`)) // "array"
// e.WriteToken(BeginArray) // [
// e.WriteToken(Null) // null
// e.WriteToken(False) // false
// e.WriteValue(Value("true")) // true
// e.WriteToken(Float(3.14159)) // 3.14159
// e.WriteToken(EndArray) // ]
// e.WriteValue(Value(`"object"`)) // "object"
// e.WriteValue(Value(`{"k":"v"}`)) // {"k":"v"}
// e.WriteToken(EndObject) // }
//
// The above is one of many possible sequences of calls and
// may not represent the most sensible method to call for any given token/value.
// For example, it is probably more common to call [Encoder.WriteToken] with a string
// for object names.
type Encoder = jsontext.Encoder
// NewEncoder constructs a new streaming encoder writing to w
// configured with the provided options.
// It flushes the internal buffer when the buffer is sufficiently full or
// when a top-level value has been written.
//
// If w is a [bytes.Buffer], then the encoder appends directly into the buffer
// without copying the contents from an intermediate buffer.
func NewEncoder(w io.Writer, opts ...Options) *Encoder {
return jsontext.NewEncoder(w, opts...)
}
// SyntacticError is a description of a syntactic error that occurred when
// encoding or decoding JSON according to the grammar.
//
// The contents of this error as produced by this package may change over time.
type SyntacticError = jsontext.SyntacticError
// Options configures [NewEncoder], [Encoder.Reset], [NewDecoder],
// and [Decoder.Reset] with specific features.
// Each function takes in a variadic list of options, where properties
// set in later options override the value of previously set properties.
//
// There is a single Options type, which is used with both encoding and decoding.
// Some options affect both operations, while others only affect one operation:
//
// - [AllowDuplicateNames] affects encoding and decoding
// - [AllowInvalidUTF8] affects encoding and decoding
// - [EscapeForHTML] affects encoding only
// - [EscapeForJS] affects encoding only
// - [PreserveRawStrings] affects encoding only
// - [CanonicalizeRawInts] affects encoding only
// - [CanonicalizeRawFloats] affects encoding only
// - [ReorderRawObjects] affects encoding only
// - [SpaceAfterColon] affects encoding only
// - [SpaceAfterComma] affects encoding only
// - [Multiline] affects encoding only
// - [WithIndent] affects encoding only
// - [WithIndentPrefix] affects encoding only
//
// Options that do not affect a particular operation are ignored.
//
// The Options type is identical to [encoding/json.Options] and
// [encoding/json/v2.Options]. Options from the other packages may
// be passed to functionality in this package, but are ignored.
// Options from this package may be used with the other packages.
type Options = jsontext.Options
// AllowDuplicateNames specifies that JSON objects may contain
// duplicate member names. Disabling the duplicate name check may provide
// performance benefits, but breaks compliance with RFC 7493, section 2.3.
// The input or output will still be compliant with RFC 8259,
// which leaves the handling of duplicate names as unspecified behavior.
//
// This affects either encoding or decoding.
func AllowDuplicateNames(v bool) Options {
return jsontext.AllowDuplicateNames(v)
}
// AllowInvalidUTF8 specifies that JSON strings may contain invalid UTF-8,
// which will be mangled as the Unicode replacement character, U+FFFD.
// This causes the encoder or decoder to break compliance with
// RFC 7493, section 2.1, and RFC 8259, section 8.1.
//
// This affects either encoding or decoding.
func AllowInvalidUTF8(v bool) Options {
return jsontext.AllowInvalidUTF8(v)
}
// EscapeForHTML specifies that '<', '>', and '&' characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u003c) so that
// the output is safe to embed within HTML.
//
// This only affects encoding and is ignored when decoding.
func EscapeForHTML(v bool) Options {
return jsontext.EscapeForHTML(v)
}
// EscapeForJS specifies that U+2028 and U+2029 characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u2028) so that
// the output is valid to embed within JavaScript. See RFC 8259, section 12.
//
// This only affects encoding and is ignored when decoding.
func EscapeForJS(v bool) Options {
return jsontext.EscapeForJS(v)
}
// PreserveRawStrings specifies that when encoding a raw JSON string in a
// [Token] or [Value], pre-escaped sequences
// in a JSON string are preserved to the output.
// However, raw strings still respect [EscapeForHTML] and [EscapeForJS]
// such that the relevant characters are escaped.
// If [AllowInvalidUTF8] is enabled, bytes of invalid UTF-8
// are preserved to the output.
//
// This only affects encoding and is ignored when decoding.
func PreserveRawStrings(v bool) Options {
return jsontext.PreserveRawStrings(v)
}
// CanonicalizeRawInts specifies that when encoding a raw JSON
// integer number (i.e., a number without a fraction and exponent) in a
// [Token] or [Value], the number is canonicalized
// according to RFC 8785, section 3.2.2.3. As a special case,
// the number -0 is canonicalized as 0.
//
// JSON numbers are treated as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example,
// integer values beyond ±2⁵³ will lose their precision.
// For example, 1234567890123456789 is formatted as 1234567890123456800.
//
// This only affects encoding and is ignored when decoding.
func CanonicalizeRawInts(v bool) Options {
return jsontext.CanonicalizeRawInts(v)
}
// CanonicalizeRawFloats specifies that when encoding a raw JSON
// floating-point number (i.e., a number with a fraction or exponent) in a
// [Token] or [Value], the number is canonicalized
// according to RFC 8785, section 3.2.2.3. As a special case,
// the number -0 is canonicalized as 0.
//
// JSON numbers are treated as IEEE 754 double precision numbers.
// It is safe to canonicalize a serialized single precision number and
// parse it back as a single precision number and expect the same value.
// If a number exceeds ±1.7976931348623157e+308, which is the maximum
// finite number, then it is saturated at that value and formatted as such.
//
// This only affects encoding and is ignored when decoding.
func CanonicalizeRawFloats(v bool) Options {
return jsontext.CanonicalizeRawFloats(v)
}
// ReorderRawObjects specifies that when encoding a raw JSON object in a
// [Value], the object members are reordered according to
// RFC 8785, section 3.2.3.
//
// This only affects encoding and is ignored when decoding.
func ReorderRawObjects(v bool) Options {
return jsontext.ReorderRawObjects(v)
}
// SpaceAfterColon specifies that the JSON output should emit a space character
// after each colon separator following a JSON object name.
// If false, then no space character appears after the colon separator.
//
// This only affects encoding and is ignored when decoding.
func SpaceAfterColon(v bool) Options {
return jsontext.SpaceAfterColon(v)
}
// SpaceAfterComma specifies that the JSON output should emit a space character
// after each comma separator following a JSON object value or array element.
// If false, then no space character appears after the comma separator.
//
// This only affects encoding and is ignored when decoding.
func SpaceAfterComma(v bool) Options {
return jsontext.SpaceAfterComma(v)
}
// Multiline specifies that the JSON output should expand to multiple lines,
// where every JSON object member or JSON array element appears on
// a new, indented line according to the nesting depth.
//
// If [SpaceAfterColon] is not specified, then the default is true.
// If [SpaceAfterComma] is not specified, then the default is false.
// If [WithIndent] is not specified, then the default is "\t".
//
// If set to false, then the output is a single-line,
// where the only whitespace emitted is determined by the current
// values of [SpaceAfterColon] and [SpaceAfterComma].
//
// This only affects encoding and is ignored when decoding.
func Multiline(v bool) Options {
return jsontext.Multiline(v)
}
// WithIndent specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix (see [WithIndentPrefix])
// followed by one or more copies of indent according to the nesting depth.
// The indent must only be composed of space or tab characters.
//
// If the intent is to emit indented output without a preference for
// the particular indent string, then use [Multiline] instead.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Multiline] being set to true.
func WithIndent(indent string) Options {
return jsontext.WithIndent(indent)
}
// WithIndentPrefix specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix followed by one or more copies of indent
// (see [WithIndent]) according to the nesting depth.
// The prefix must only be composed of space or tab characters.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Multiline] being set to true.
func WithIndentPrefix(prefix string) Options {
return jsontext.WithIndentPrefix(prefix)
}
// AppendQuote appends a double-quoted JSON string literal representing src
// to dst and returns the extended buffer.
// It uses the minimal string representation per RFC 8785, section 3.2.2.2.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
// The dst must not overlap with the src.
func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
return jsontext.AppendQuote[Bytes](dst, src)
}
// AppendUnquote appends the decoded interpretation of src as a
// double-quoted JSON string literal to dst and returns the extended buffer.
// The input src must be a JSON string without any surrounding whitespace.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
// Any trailing bytes after the JSON string literal results in an error.
// The dst must not overlap with the src.
func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
return jsontext.AppendUnquote[Bytes](dst, src)
}
// ErrDuplicateName indicates that a JSON token could not be
// encoded or decoded because it results in a duplicate JSON object name.
// This error is directly wrapped within a [SyntacticError] when produced.
//
// The name of a duplicate JSON object member can be extracted as:
//
// err := ...
// serr, ok := errors.AsType[*jsontext.SyntacticError](err)
// if ok && serr.Err == jsontext.ErrDuplicateName {
// ptr := serr.JSONPointer // JSON pointer to duplicate name
// name := ptr.LastToken() // duplicate name itself
// ...
// }
//
// This error is only returned if [AllowDuplicateNames] is false.
var ErrDuplicateName = jsontext.ErrDuplicateName
// ErrNonStringName indicates that a JSON token could not be
// encoded or decoded because it is not a string,
// as required for JSON object names according to RFC 8259, section 4.
// This error is directly wrapped within a [SyntacticError] when produced.
var ErrNonStringName = jsontext.ErrNonStringName
// Pointer is a JSON Pointer (RFC 6901) that references a particular JSON value
// relative to the root of the top-level JSON value.
//
// A Pointer is a slash-separated list of tokens, where each token is
// either a JSON object name or an index to a JSON array element
// encoded as a base-10 integer value.
// It is impossible to distinguish between an array index and an object name
// (that happens to be a base-10 encoded integer) without also knowing
// the structure of the top-level JSON value that the pointer refers to.
//
// There is exactly one representation of a pointer to a particular value,
// so comparability of Pointer values is equivalent to checking whether
// they both point to the exact same value.
type Pointer = jsontext.Pointer
// Token represents a lexical JSON token, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a begin or end delimiter for a JSON object (i.e., { or } )
// - a begin or end delimiter for a JSON array (i.e., [ or ] )
//
// A Token cannot represent entire array or object values, while a [Value] can.
// There is no Token to represent commas and colons since
// these structural tokens can be inferred from the surrounding context.
//
// A Token stores data in one of two forms:
//
// - As raw JSON text: backed by the internal buffer of the [Decoder]
// and only ever produced by [Decoder.ReadToken].
// Such a token is only valid until the next call to any method on that
// [Decoder] (e.g., [Decoder.PeekKind], [Decoder.ReadToken],
// [Decoder.ReadValue], or [Decoder.SkipValue]).
// Call [Token.Clone] to copy the raw text into an independent allocation
// that persists beyond subsequent [Decoder] calls.
//
// - As a typed Go value: a self-contained representation produced by
// the constructor functions (e.g., [String], [Int], [Uint], [Float]).
// Such tokens are valid indefinitely and do not need to be cloned.
type Token = jsontext.Token
var (
Null = jsontext.Null
False = jsontext.False
True = jsontext.True
BeginObject = jsontext.BeginObject
EndObject = jsontext.EndObject
BeginArray = jsontext.BeginArray
EndArray = jsontext.EndArray
)
// Bool constructs a Token representing a JSON boolean.
func Bool(b bool) Token {
return jsontext.Bool(b)
}
// String constructs a Token representing a JSON string.
// The provided string should contain valid UTF-8, otherwise invalid characters
// may be mangled as the Unicode replacement character.
func String(s string) Token {
return jsontext.String(s)
}
// Float constructs a Token representing a JSON number as
// a 64-bit floating-point number formatted according to
// ECMA-262, 10th edition, section 7.1.12.1 and RFC 8785, section 3.2.2.3.
// with the exception that -0 is still formatted as -0.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
return jsontext.Float(n)
}
// Int constructs a Token representing a JSON number from an int64.
func Int(n int64) Token {
return jsontext.Int(n)
}
// Uint constructs a Token representing a JSON number from a uint64.
func Uint(n uint64) Token {
return jsontext.Uint(n)
}
// A Kind represents the kind of a JSON token.
//
// Kind represents each possible JSON token kind with a single byte,
// which is conveniently the first byte of that kind's grammar
// with the restriction that numbers always be represented with '0'.
type Kind = jsontext.Kind
const (
KindInvalid = jsontext.KindInvalid
KindNull = jsontext.KindNull
KindFalse = jsontext.KindFalse
KindTrue = jsontext.KindTrue
KindString = jsontext.KindString
KindNumber = jsontext.KindNumber
KindBeginObject = jsontext.KindBeginObject
KindEndObject = jsontext.KindEndObject
KindBeginArray = jsontext.KindBeginArray
KindEndArray = jsontext.KindEndArray
)
// AppendFormat formats the JSON value in src and appends it to dst
// according to the specified options.
// See [Value.Format] for more details about the formatting behavior.
//
// The dst and src may overlap.
// If an error is reported, then the entirety of src is appended to dst.
func AppendFormat[Bytes ~[]byte | ~string](dst []byte, src Bytes, opts ...Options) ([]byte, error) {
return jsontext.AppendFormat(dst, []byte(src), opts...)
}
// Value represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - an entire JSON object (e.g., {"fizz":"buzz"} )
// - an entire JSON array (e.g., [1,2,3] )
//
// Value can represent entire array or object values, while [Token] cannot.
// Value may contain leading and/or trailing whitespace.
type Value = jsontext.Value

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,111 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
// Package jsontext implements syntactic processing of JSON
// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
// The [Encoder] and [Decoder] types are used to encode or decode
// a stream of JSON tokens or values.
//
// # Tokens and Values
//
// A JSON token refers to the basic structural elements of JSON:
//
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a begin or end delimiter for a JSON object (i.e., '{' or '}')
// - a begin or end delimiter for a JSON array (i.e., '[' or ']')
//
// A JSON token is represented by the [Token] type in Go. Technically,
// there are two additional structural characters (i.e., ':' and ','),
// but there is no [Token] representation for them since their presence
// can be inferred by the structure of the JSON grammar itself.
// For example, there must always be an implicit colon between
// the name and value of a JSON object member.
//
// A JSON value refers to a complete unit of JSON data:
//
// - a JSON literal, string, or number
// - a JSON object (e.g., `{"name":"value"}`)
// - a JSON array (e.g., `[1,2,3]`)
//
// A JSON value is represented by the [Value] type in Go and is a []byte
// containing the raw textual representation of the value. There is some overlap
// between tokens and values as both contain literals, strings, and numbers.
// However, only a value can represent the entirety of a JSON object or array.
//
// The [Encoder] and [Decoder] types contain methods to read or write the next
// [Token] or [Value] in a sequence. They maintain a state machine to validate
// whether the sequence of JSON tokens and/or values produces a valid JSON.
// [Options] may be passed to the [NewEncoder] or [NewDecoder] constructors
// to configure the syntactic behavior of encoding and decoding.
//
// # Terminology
//
// The terms "encode" and "decode" are used for syntactic functionality
// that is concerned with processing JSON based on its grammar, and
// the terms "marshal" and "unmarshal" are used for semantic functionality
// that determines the meaning of JSON values as Go values and vice-versa.
// This package (i.e., [jsontext]) deals with JSON at a syntactic layer,
// while [encoding/json/v2] deals with JSON at a semantic layer.
// The goal is to provide a clear distinction between functionality that
// is purely concerned with encoding versus that of marshaling.
// For example, one can directly encode a stream of JSON tokens without
// needing to marshal a concrete Go value representing them.
// Similarly, one can decode a stream of JSON tokens without
// needing to unmarshal them into a concrete Go value.
//
// This package uses JSON terminology when discussing JSON, which may differ
// from related concepts in Go or elsewhere in computing literature.
//
// - a JSON "object" refers to an unordered collection of name/value members.
// - a JSON "array" refers to an ordered sequence of elements.
// - a JSON "value" refers to either a literal (i.e., null, false, or true),
// string, number, object, or array.
//
// See RFC 8259 for more information.
//
// # Specifications
//
// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259,
// and RFC 8785. Each RFC is generally a stricter subset of another RFC.
// In increasing order of strictness:
//
// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8
// and also do not require (but recommend) that object names be unique.
// - RFC 8259 requires the use of UTF-8,
// but does not require (but recommends) that object names be unique.
// - RFC 7493 requires the use of UTF-8
// and also requires that object names be unique.
// - RFC 8785 defines a canonical representation. It requires the use of UTF-8
// and also requires that object names be unique and in a specific ordering.
// It specifies exactly how strings and numbers must be formatted.
//
// The primary difference between RFC 4627 and RFC 7159 is that the former
// restricted top-level values to only JSON objects and arrays, while
// RFC 7159 and subsequent RFCs permit top-level values to additionally be
// JSON nulls, booleans, strings, or numbers.
//
// By default, this package operates on RFC 7493, but can be configured
// to operate according to the other RFC specifications.
// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it.
// In particular, it makes specific choices about behavior that RFC 8259
// leaves as undefined in order to ensure greater interoperability.
//
// # Security Considerations
//
// See the "Security Considerations" section in [encoding/json/v2].
package jsontext
// requireKeyedLiterals can be embedded in a struct to require keyed literals.
type requireKeyedLiterals struct{}
// nonComparable can be embedded in a struct to prevent comparability.
type nonComparable [0]func()

View File

@@ -0,0 +1,993 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"bytes"
"io"
"math/bits"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// Encoder is a streaming encoder from raw JSON tokens and values.
// It is used to write a stream of top-level JSON values,
// each terminated with a newline character.
//
// [Encoder.WriteToken] and [Encoder.WriteValue] calls may be interleaved.
// For example, the following JSON value:
//
// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}}
//
// can be composed with the following calls (ignoring errors for brevity):
//
// e.WriteToken(BeginObject) // {
// e.WriteToken(String("name")) // "name"
// e.WriteToken(String("value")) // "value"
// e.WriteValue(Value(`"array"`)) // "array"
// e.WriteToken(BeginArray) // [
// e.WriteToken(Null) // null
// e.WriteToken(False) // false
// e.WriteValue(Value("true")) // true
// e.WriteToken(Float(3.14159)) // 3.14159
// e.WriteToken(EndArray) // ]
// e.WriteValue(Value(`"object"`)) // "object"
// e.WriteValue(Value(`{"k":"v"}`)) // {"k":"v"}
// e.WriteToken(EndObject) // }
//
// The above is one of many possible sequences of calls and
// may not represent the most sensible method to call for any given token/value.
// For example, it is probably more common to call [Encoder.WriteToken] with a string
// for object names.
type Encoder struct {
s encoderState
}
// encoderState is the low-level state of Encoder.
// It has exported fields and methods for use by the "json" package.
type encoderState struct {
state
encodeBuffer
jsonopts.Struct
SeenPointers map[any]struct{} // only used when marshaling; identical to json.seenPointers
}
// encodeBuffer is a buffer split into 2 segments:
//
// - buf[0:len(buf)] // written (but unflushed) portion of the buffer
// - buf[len(buf):cap(buf)] // unused portion of the buffer
type encodeBuffer struct {
Buf []byte // may alias wr if it is a bytes.Buffer
// baseOffset is added to len(buf) to obtain the absolute offset
// relative to the start of io.Writer stream.
baseOffset int64
wr io.Writer
// maxValue is the approximate maximum Value size passed to WriteValue.
maxValue int
// availBuffer is the buffer returned by the AvailableBuffer method.
availBuffer []byte // always has zero length
// bufStats is statistics about buffer utilization.
// It is only used with pooled encoders in pools.go.
bufStats bufferStatistics
}
// NewEncoder constructs a new streaming encoder writing to w
// configured with the provided options.
// It flushes the internal buffer when the buffer is sufficiently full or
// when a top-level value has been written.
//
// If w is a [bytes.Buffer], then the encoder appends directly into the buffer
// without copying the contents from an intermediate buffer.
func NewEncoder(w io.Writer, opts ...Options) *Encoder {
e := new(Encoder)
e.Reset(w, opts...)
return e
}
// Reset resets an encoder such that it is writing afresh to w and
// configured with the provided options. Reset must not be called on
// an Encoder passed to the [encoding/json/v2.MarshalerTo.MarshalJSONTo] method
// or the [encoding/json/v2.MarshalToFunc] function.
func (e *Encoder) Reset(w io.Writer, opts ...Options) {
switch {
case e == nil:
panic("jsontext: invalid nil Encoder")
case w == nil:
panic("jsontext: invalid nil io.Writer")
case e.s.Flags.Get(jsonflags.WithinArshalCall):
panic("jsontext: cannot reset Encoder passed to json.MarshalerTo")
}
// Reuse the buffer if it does not alias a previous [bytes.Buffer].
b := e.s.Buf[:0]
if _, ok := e.s.wr.(*bytes.Buffer); ok {
b = nil
}
e.s.reset(b, w, opts...)
}
func (e *encoderState) reset(b []byte, w io.Writer, opts ...Options) {
e.state.reset()
e.encodeBuffer = encodeBuffer{Buf: b, wr: w, availBuffer: e.availBuffer, bufStats: e.bufStats}
if bb, ok := w.(*bytes.Buffer); ok && bb != nil {
e.Buf = bb.AvailableBuffer() // alias the unused buffer of bb
}
opts2 := jsonopts.Struct{} // avoid mutating e.Struct in case it is part of opts
opts2.Join(opts...)
e.Struct = opts2
if e.Struct.Flags.Get(jsonflags.Multiline) {
e.Struct.InitializeMultiline()
}
}
// Options returns the options used to construct the encoder and
// may additionally contain semantic options passed to a
// [encoding/json/v2.MarshalEncode] call.
//
// If operating within
// a [encoding/json/v2.MarshalerTo.MarshalJSONTo] method call or
// a [encoding/json/v2.MarshalToFunc] function call,
// then the returned options are only valid within the call.
func (e *Encoder) Options() Options {
return &e.s.Struct
}
func (e *encoderState) options() *jsonopts.Struct { return &e.Struct }
// NeedFlush determines whether to flush at this point.
func (e *encoderState) NeedFlush() bool {
// NOTE: This function is carefully written to be inlinable.
// Avoid flushing if e.wr is nil since there is no underlying writer.
// Flush if less than 25% of the capacity remains.
// Flushing at some constant fraction ensures that the buffer stops growing
// so long as the largest Token or Value fits within that unused capacity.
return e.wr != nil && (e.Tokens.Depth() == 1 || len(e.Buf) > 3*cap(e.Buf)/4)
}
// Flush flushes the buffer to the underlying io.Writer.
// It may append a trailing newline after the top-level value.
func (e *encoderState) Flush() error {
if e.wr == nil || e.avoidFlush() {
return nil
}
// In streaming mode, always emit a newline after the top-level value.
if e.Tokens.Depth() == 1 && !e.Flags.Get(jsonflags.OmitTopLevelNewline) {
e.Buf = append(e.Buf, '\n')
}
// Inform objectNameStack that we are about to flush the buffer content.
e.Names.copyQuotedBuffer(e.Buf)
// Specialize bytes.Buffer for better performance.
if bb, ok := e.wr.(*bytes.Buffer); ok {
// If e.Buf already aliases the internal buffer of bb,
// then the Write call simply increments the internal offset,
// otherwise Write operates as expected.
// See https://go.dev/issue/42986.
n, _ := bb.Write(e.Buf) // never fails unless bb is nil
e.baseOffset += int64(n)
// If the internal buffer of bytes.Buffer is too small,
// append operations elsewhere in the Encoder may grow the buffer.
// This would be semantically correct, but hurts performance.
// As such, ensure 25% of the current length is always available
// to reduce the probability that other appends must allocate.
if avail := bb.Available(); avail < bb.Len()/4 {
bb.Grow(avail + 1)
}
e.Buf = bb.AvailableBuffer()
return nil
}
// Flush the internal buffer to the underlying io.Writer.
n, err := e.wr.Write(e.Buf)
e.baseOffset += int64(n)
if err != nil {
// In the event of an error, preserve the unflushed portion.
// Thus, write errors aren't fatal so long as the io.Writer
// maintains consistent state after errors.
if n > 0 {
e.Buf = e.Buf[:copy(e.Buf, e.Buf[n:])]
}
return &ioError{action: "write", err: err}
}
e.Buf = e.Buf[:0]
// Check whether to grow the buffer.
// Note that cap(e.Buf) may already exceed maxBufferSize since
// an append elsewhere already grew it to store a large token.
const maxBufferSize = 4 << 10
const growthSizeFactor = 2 // higher value is faster
const growthRateFactor = 2 // higher value is slower
// By default, grow if below the maximum buffer size.
grow := cap(e.Buf) <= maxBufferSize/growthSizeFactor
// Growing can be expensive, so only grow
// if a sufficient number of bytes have been processed.
grow = grow && int64(cap(e.Buf)) < e.previousOffsetEnd()/growthRateFactor
if grow {
e.Buf = make([]byte, 0, cap(e.Buf)*growthSizeFactor)
}
return nil
}
func (e *encodeBuffer) offsetAt(pos int) int64 { return e.baseOffset + int64(pos) }
func (e *encodeBuffer) previousOffsetEnd() int64 { return e.baseOffset + int64(len(e.Buf)) }
func (e *encodeBuffer) unflushedBuffer() []byte { return e.Buf }
// avoidFlush indicates whether to avoid flushing to ensure there is always
// enough in the buffer to unwrite the last object member if it were empty.
func (e *encoderState) avoidFlush() bool {
switch {
case e.Tokens.Last.Length() == 0:
// Never flush after BeginObject or BeginArray since we don't know yet
// if the object or array will end up being empty.
return true
case e.Tokens.Last.needObjectValue():
// Never flush before the object value since we don't know yet
// if the object value will end up being empty.
return true
case e.Tokens.Last.NeedObjectName() && len(e.Buf) >= 2:
// Never flush after the object value if it does turn out to be empty.
switch string(e.Buf[len(e.Buf)-2:]) {
case `ll`, `""`, `{}`, `[]`: // last two bytes of every empty value
return true
}
}
return false
}
// UnwriteEmptyObjectMember unwrites the last object member if it is empty
// and reports whether it performed an unwrite operation.
func (e *encoderState) UnwriteEmptyObjectMember(prevName *string) bool {
if last := e.Tokens.Last; !last.isObject() || !last.NeedObjectName() || last.Length() == 0 {
panic("BUG: must be called on an object after writing a value")
}
// The flushing logic is modified to never flush a trailing empty value.
// The encoder never writes trailing whitespace eagerly.
b := e.unflushedBuffer()
// Detect whether the last value was empty.
var n int
if len(b) >= 3 {
switch string(b[len(b)-2:]) {
case "ll": // last two bytes of `null`
n = len(`null`)
case `""`:
// It is possible for a non-empty string to have `""` as a suffix
// if the second to the last quote was escaped.
if b[len(b)-3] == '\\' {
return false // e.g., `"\""` is not empty
}
n = len(`""`)
case `{}`:
n = len(`{}`)
case `[]`:
n = len(`[]`)
}
}
if n == 0 {
return false
}
// Unwrite the value, whitespace, colon, name, whitespace, and comma.
b = b[:len(b)-n]
b = jsonwire.TrimSuffixWhitespace(b)
b = jsonwire.TrimSuffixByte(b, ':')
b = jsonwire.TrimSuffixString(b)
b = jsonwire.TrimSuffixWhitespace(b)
b = jsonwire.TrimSuffixByte(b, ',')
e.Buf = b // store back truncated unflushed buffer
// Undo state changes.
e.Tokens.Last.decrement() // for object member value
e.Tokens.Last.decrement() // for object member name
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if e.Tokens.Last.isActiveNamespace() {
e.Namespaces.Last().removeLast()
}
}
e.Names.clearLast()
if prevName != nil {
e.Names.copyQuotedBuffer(e.Buf) // required by objectNameStack.replaceLastUnquotedName
e.Names.replaceLastUnquotedName(*prevName)
}
return true
}
// UnwriteOnlyObjectMemberName unwrites the only object member name
// and returns the unquoted name.
func (e *encoderState) UnwriteOnlyObjectMemberName() string {
if last := e.Tokens.Last; !last.isObject() || last.Length() != 1 {
panic("BUG: must be called on an object after writing first name")
}
// Unwrite the name and whitespace.
b := jsonwire.TrimSuffixString(e.Buf)
isVerbatim := bytes.IndexByte(e.Buf[len(b):], '\\') < 0
name := string(jsonwire.UnquoteMayCopy(e.Buf[len(b):], isVerbatim))
e.Buf = jsonwire.TrimSuffixWhitespace(b)
// Undo state changes.
e.Tokens.Last.decrement()
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if e.Tokens.Last.isActiveNamespace() {
e.Namespaces.Last().removeLast()
}
}
e.Names.clearLast()
return name
}
// WriteToken writes the next token and advances the internal write offset.
//
// The provided token kind must be consistent with the JSON grammar.
// For example, it is an error to provide a number when the encoder
// is expecting an object name (which is always a string), or
// to provide an end object delimiter when the encoder is finishing an array.
// If the provided token is invalid, then it reports a [SyntacticError] and
// the internal state remains unchanged. The offset reported
// in [SyntacticError] will be the [Encoder.OutputOffset] plus any delimiter
// or whitespace characters that would have preceded the provided token.
func (e *Encoder) WriteToken(t Token) error {
return e.s.WriteToken(t)
}
func (e *encoderState) WriteToken(t Token) error {
k := t.Kind()
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.AnyWhitespace) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the token
// Append the token to the output and to the state machine.
var err error
switch k {
case 'n':
b = append(b, "null"...)
err = e.Tokens.appendLiteral()
case 'f':
b = append(b, "false"...)
err = e.Tokens.appendLiteral()
case 't':
b = append(b, "true"...)
err = e.Tokens.appendLiteral()
case '"':
if b, err = t.appendString(b, &e.Flags); err != nil {
break
}
if e.Tokens.Last.NeedObjectName() {
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if !e.Tokens.Last.isValidNamespace() {
err = errInvalidNamespace
break
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
err = wrapWithObjectName(ErrDuplicateName, b[pos:])
break
}
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
err = e.Tokens.appendString()
case '0':
if b, err = t.appendNumber(b, &e.Flags); err != nil {
break
}
err = e.Tokens.appendNumber()
case '{':
b = append(b, '{')
if err = e.Tokens.pushObject(); err != nil {
break
}
e.Names.push()
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Namespaces.push()
}
e.Flags.Clear(jsonflags.TagFlags) // tags only apply to current depth
case '}':
b = append(b, '}')
if err = e.Tokens.popObject(); err != nil {
break
}
e.Names.pop()
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Namespaces.pop()
}
case '[':
b = append(b, '[')
err = e.Tokens.pushArray()
e.Flags.Clear(jsonflags.TagFlags) // tags only apply to current depth
case ']':
b = append(b, ']')
err = e.Tokens.popArray()
default:
err = errInvalidToken
}
if err != nil {
return wrapSyntacticError(e, err, pos, +1)
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// AppendRaw appends either a raw string (without double quotes) or number.
// Specify safeASCII if the string output is guaranteed to be ASCII
// without any characters (including '<', '>', and '&') that need escaping,
// otherwise this will validate whether the string needs escaping.
// The appended bytes for a JSON number must be valid.
//
// This is a specialized implementation of Encoder.WriteValue
// that allows appending directly into the buffer.
// It is only called from marshal logic in the "json" package.
func (e *encoderState) AppendRaw(k Kind, safeASCII bool, appendFn func([]byte) ([]byte, error)) error {
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.AnyWhitespace) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the token
var err error
switch k {
case '"':
// Append directly into the encoder buffer by assuming that
// most of the time none of the characters need escaping.
b = append(b, '"')
if b, err = appendFn(b); err != nil {
return err
}
b = append(b, '"')
// Check whether we need to escape the string and if necessary
// copy it to a scratch buffer and then escape it back.
isVerbatim := safeASCII || !jsonwire.NeedEscape(b[pos+len(`"`):len(b)-len(`"`)])
if !isVerbatim {
var err error
b2 := append(e.availBuffer, b[pos+len(`"`):len(b)-len(`"`)]...)
b, err = jsonwire.AppendQuote(b[:pos], b2, &e.Flags)
e.availBuffer = b2[:0]
if err != nil {
return wrapSyntacticError(e, err, pos, +1)
}
}
// Update the state machine.
if e.Tokens.Last.NeedObjectName() {
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if !e.Tokens.Last.isValidNamespace() {
return wrapSyntacticError(e, errInvalidNamespace, pos, +1)
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], isVerbatim) {
err = wrapWithObjectName(ErrDuplicateName, b[pos:])
return wrapSyntacticError(e, err, pos, +1)
}
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
if err := e.Tokens.appendString(); err != nil {
return wrapSyntacticError(e, err, pos, +1)
}
case '0':
if b, err = appendFn(b); err != nil {
return err
}
if err := e.Tokens.appendNumber(); err != nil {
return wrapSyntacticError(e, err, pos, +1)
}
default:
panic("BUG: invalid kind")
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// WriteValue writes the next raw value and advances the internal write offset.
// The Encoder does not simply copy the provided value verbatim, but
// parses it to ensure that it is syntactically valid and reformats it
// according to how the Encoder is configured to format whitespace and strings.
// If [AllowInvalidUTF8] is specified, then any invalid UTF-8 is mangled
// as the Unicode replacement character, U+FFFD.
//
// The provided value kind must be consistent with the JSON grammar
// (see examples on [Encoder.WriteToken]). If the provided value is invalid,
// then it reports a [SyntacticError] and the internal state remains unchanged.
// The offset reported in [SyntacticError] will be the [Encoder.OutputOffset]
// plus the offset into v of any encountered syntax error.
func (e *Encoder) WriteValue(v Value) error {
return e.s.WriteValue(v)
}
func (e *encoderState) WriteValue(v Value) error {
e.maxValue |= len(v) // bitwise OR is a fast approximation of max
k := v.Kind()
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.AnyWhitespace) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the value
// Append the value to the output.
var n int
n += jsonwire.ConsumeWhitespace(v[n:])
b, m, err := e.reformatValue(b, v[n:], e.Tokens.Depth())
if err != nil {
return wrapSyntacticError(e, err, pos+n+m, +1)
}
n += m
n += jsonwire.ConsumeWhitespace(v[n:])
if len(v) > n {
err = jsonwire.NewInvalidCharacterError(v[n:], "after top-level value")
return wrapSyntacticError(e, err, pos+n, 0)
}
// Append the kind to the state machine.
switch k {
case 'n', 'f', 't':
err = e.Tokens.appendLiteral()
case '"':
if e.Tokens.Last.NeedObjectName() {
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if !e.Tokens.Last.isValidNamespace() {
err = errInvalidNamespace
break
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
err = wrapWithObjectName(ErrDuplicateName, b[pos:])
break
}
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
err = e.Tokens.appendString()
case '0':
err = e.Tokens.appendNumber()
case '{':
if err = e.Tokens.pushObject(); err != nil {
break
}
if err = e.Tokens.popObject(); err != nil {
panic("BUG: popObject should never fail immediately after pushObject: " + err.Error())
}
if e.Flags.Get(jsonflags.ReorderRawObjects) {
mustReorderObjects(b[pos:])
}
case '[':
if err = e.Tokens.pushArray(); err != nil {
break
}
if err = e.Tokens.popArray(); err != nil {
panic("BUG: popArray should never fail immediately after pushArray: " + err.Error())
}
if e.Flags.Get(jsonflags.ReorderRawObjects) {
mustReorderObjects(b[pos:])
}
}
if err != nil {
return wrapSyntacticError(e, err, pos, +1)
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// CountNextDelimWhitespace counts the number of bytes of delimiter and
// whitespace bytes assuming the upcoming token is a JSON value.
// This method is used for error reporting at the semantic layer.
func (e *encoderState) CountNextDelimWhitespace() (n int) {
const next = Kind('"') // arbitrary kind as next JSON value
delim := e.Tokens.needDelim(next)
if delim > 0 {
n += len(",") | len(":")
}
if delim == ':' {
if e.Flags.Get(jsonflags.SpaceAfterColon) {
n += len(" ")
}
} else {
if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) {
n += len(" ")
}
if e.Flags.Get(jsonflags.Multiline) {
if m := e.Tokens.NeedIndent(next); m > 0 {
n += len("\n") + len(e.IndentPrefix) + (m-1)*len(e.Indent)
}
}
}
return n
}
// appendWhitespace appends whitespace that immediately precedes the next token.
func (e *encoderState) appendWhitespace(b []byte, next Kind) []byte {
if delim := e.Tokens.needDelim(next); delim == ':' {
if e.Flags.Get(jsonflags.SpaceAfterColon) {
b = append(b, ' ')
}
} else {
if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) {
b = append(b, ' ')
}
if e.Flags.Get(jsonflags.Multiline) {
b = e.AppendIndent(b, e.Tokens.NeedIndent(next))
}
}
return b
}
// AppendIndent appends the appropriate number of indentation characters
// for the current nested level, n.
func (e *encoderState) AppendIndent(b []byte, n int) []byte {
if n == 0 {
return b
}
b = append(b, '\n')
b = append(b, e.IndentPrefix...)
for ; n > 1; n-- {
b = append(b, e.Indent...)
}
return b
}
// reformatValue parses a JSON value from the start of src and
// appends it to the end of dst, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatValue(dst []byte, src Value, depth int) ([]byte, int, error) {
// TODO: Should this update ValueFlags as input?
if len(src) == 0 {
return dst, 0, io.ErrUnexpectedEOF
}
switch k := Kind(src[0]).normalize(); k {
case 'n':
if jsonwire.ConsumeNull(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "null")
return dst, n, err
}
return append(dst, "null"...), len("null"), nil
case 'f':
if jsonwire.ConsumeFalse(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "false")
return dst, n, err
}
return append(dst, "false"...), len("false"), nil
case 't':
if jsonwire.ConsumeTrue(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "true")
return dst, n, err
}
return append(dst, "true"...), len("true"), nil
case '"':
if n := jsonwire.ConsumeSimpleString(src); n > 0 {
dst = append(dst, src[:n]...) // copy simple strings verbatim
return dst, n, nil
}
return jsonwire.ReformatString(dst, src, &e.Flags)
case '0':
if n := jsonwire.ConsumeSimpleNumber(src); n > 0 && !e.Flags.Get(jsonflags.CanonicalizeNumbers) {
dst = append(dst, src[:n]...) // copy simple numbers verbatim
return dst, n, nil
}
return jsonwire.ReformatNumber(dst, src, &e.Flags)
case '{':
return e.reformatObject(dst, src, depth)
case '[':
return e.reformatArray(dst, src, depth)
default:
return dst, 0, jsonwire.NewInvalidCharacterError(src, "at start of value")
}
}
// reformatObject parses a JSON object from the start of src and
// appends it to the end of dst, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatObject(dst []byte, src Value, depth int) ([]byte, int, error) {
// Append object begin.
if len(src) == 0 || src[0] != '{' {
panic("BUG: reformatObject must be called with a buffer that starts with '{'")
} else if depth == maxNestingDepth+1 {
return dst, 0, errMaxDepth
}
dst = append(dst, '{')
n := len("{")
// Append (possible) object end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
if src[n] == '}' {
dst = append(dst, '}')
n += len("}")
return dst, n, nil
}
var err error
var names *objectNamespace
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Namespaces.push()
defer e.Namespaces.pop()
names = e.Namespaces.Last()
}
depth++
for {
// Append optional newline and indentation.
if e.Flags.Get(jsonflags.Multiline) {
dst = e.AppendIndent(dst, depth)
}
// Append object name.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
m := jsonwire.ConsumeSimpleString(src[n:])
isVerbatim := m > 0
if isVerbatim {
dst = append(dst, src[n:n+m]...)
} else {
dst, m, err = jsonwire.ReformatString(dst, src[n:], &e.Flags)
if err != nil {
return dst, n + m, err
}
}
quotedName := src[n : n+m]
if !e.Flags.Get(jsonflags.AllowDuplicateNames) && !names.insertQuoted(quotedName, isVerbatim) {
return dst, n, wrapWithObjectName(ErrDuplicateName, quotedName)
}
n += m
// Append colon.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName)
}
if src[n] != ':' {
err = jsonwire.NewInvalidCharacterError(src[n:], "after object name (expecting ':')")
return dst, n, wrapWithObjectName(err, quotedName)
}
dst = append(dst, ':')
n += len(":")
if e.Flags.Get(jsonflags.SpaceAfterColon) {
dst = append(dst, ' ')
}
// Append object value.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName)
}
dst, m, err = e.reformatValue(dst, src[n:], depth)
if err != nil {
return dst, n + m, wrapWithObjectName(err, quotedName)
}
n += m
// Append comma or object end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
switch src[n] {
case ',':
dst = append(dst, ',')
if e.Flags.Get(jsonflags.SpaceAfterComma) {
dst = append(dst, ' ')
}
n += len(",")
continue
case '}':
if e.Flags.Get(jsonflags.Multiline) {
dst = e.AppendIndent(dst, depth-1)
}
dst = append(dst, '}')
n += len("}")
return dst, n, nil
default:
return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after object value (expecting ',' or '}')")
}
}
}
// reformatArray parses a JSON array from the start of src and
// appends it to the end of dst, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatArray(dst []byte, src Value, depth int) ([]byte, int, error) {
// Append array begin.
if len(src) == 0 || src[0] != '[' {
panic("BUG: reformatArray must be called with a buffer that starts with '['")
} else if depth == maxNestingDepth+1 {
return dst, 0, errMaxDepth
}
dst = append(dst, '[')
n := len("[")
// Append (possible) array end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
if src[n] == ']' {
dst = append(dst, ']')
n += len("]")
return dst, n, nil
}
var idx int64
var err error
depth++
for {
// Append optional newline and indentation.
if e.Flags.Get(jsonflags.Multiline) {
dst = e.AppendIndent(dst, depth)
}
// Append array value.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
var m int
dst, m, err = e.reformatValue(dst, src[n:], depth)
if err != nil {
return dst, n + m, wrapWithArrayIndex(err, idx)
}
n += m
// Append comma or array end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
switch src[n] {
case ',':
dst = append(dst, ',')
if e.Flags.Get(jsonflags.SpaceAfterComma) {
dst = append(dst, ' ')
}
n += len(",")
idx++
continue
case ']':
if e.Flags.Get(jsonflags.Multiline) {
dst = e.AppendIndent(dst, depth-1)
}
dst = append(dst, ']')
n += len("]")
return dst, n, nil
default:
return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after array value (expecting ',' or ']')")
}
}
}
// OutputOffset returns the current output byte offset. It gives the location
// of the next byte immediately after the most recently written token or value.
// The number of bytes actually written to the underlying [io.Writer] may be less
// than this offset due to internal buffering effects.
func (e *Encoder) OutputOffset() int64 {
return e.s.previousOffsetEnd()
}
// AvailableBuffer returns a zero-length buffer with a possible non-zero capacity.
// This buffer is intended to be used to populate a [Value]
// being passed to an immediately succeeding [Encoder.WriteValue] call.
//
// Example usage:
//
// b := e.AvailableBuffer()
// b = append(b, '"')
// b = appendString(b, v) // append the string formatting of v
// b = append(b, '"')
// ... := e.WriteValue(b)
//
// WriteValue expects a JSON value. Using AvailableBuffer to manually construct
// a value requires caution to avoid producing an invalid JSON value that would
// then cause WriteValue to fail.
func (e *Encoder) AvailableBuffer() []byte {
// NOTE: We don't return e.Buf[len(e.Buf):cap(e.Buf)] since WriteValue would
// need to take special care to avoid mangling the data while reformatting.
// WriteValue can't easily identify whether the input Value aliases e.Buf
// without using unsafe.Pointer. Thus, we just return a different buffer.
// Should this ever alias e.Buf, we need to consider how it operates with
// the specialized performance optimization for bytes.Buffer.
n := 1 << bits.Len(uint(e.s.maxValue|63)) // fast approximation for max length
if cap(e.s.availBuffer) < n {
e.s.availBuffer = make([]byte, 0, n)
}
return e.s.availBuffer
}
// StackDepth returns the depth of the state machine for written JSON data.
// Each level on the stack represents a nested JSON object or array.
// It is incremented whenever a [BeginObject] or [BeginArray] token is encountered
// and decremented whenever an [EndObject] or [EndArray] token is encountered.
//
// StackDepth returns 0 when not inside any object or array.
// In particular, it returns 0 before any tokens have been written,
// after any top-level value has been written, and between values
// when encoding a stream of top-level values (e.g., NDJSON).
// StackDepth returns 1 inside a top-level object or array,
// 2 inside a nested object or array, and so on.
//
// For example, consider encoding the following JSON:
//
// {"a": [1, 2], "b": {"c": 3}}
//
// While encoding, StackDepth would report the following:
//
// - At the start, StackDepth reports 0.
// - After encoding the outer '{', StackDepth reports 1.
// - After encoding the inner '[', StackDepth reports 2.
// - After encoding the inner ']', StackDepth reports 1.
// - After encoding the outer '}', StackDepth reports 0.
func (e *Encoder) StackDepth() int {
// NOTE: Keep in sync with Decoder.StackDepth.
return e.s.Tokens.Depth() - 1
}
// StackIndex returns information about the specified stack level.
// It must be a number between 0 and [Encoder.StackDepth], inclusive.
// For each level, it reports the kind:
//
// - [KindInvalid] for a level of zero,
// - [KindBeginObject] for a level representing a JSON object, and
// - [KindBeginArray] for a level representing a JSON array.
//
// It also reports the length of that JSON object or array encoded so far.
// Each name and value in a JSON object is counted separately,
// so the effective number of members would be half the length.
// A complete JSON object must have an even length.
func (e *Encoder) StackIndex(i int) (Kind, int64) {
// NOTE: Keep in sync with Decoder.StackIndex.
switch s := e.s.Tokens.index(i); {
case i > 0 && s.isObject():
return '{', s.Length()
case i > 0 && s.isArray():
return '[', s.Length()
default:
return 0, s.Length()
}
}
// StackPointer returns a JSON Pointer (RFC 6901) to the most recently written value.
func (e *Encoder) StackPointer() Pointer {
return Pointer(e.s.AppendStackPointer(nil, -1))
}
func (e *encoderState) AppendStackPointer(b []byte, where int) []byte {
e.Names.copyQuotedBuffer(e.Buf)
return e.state.appendStackPointer(b, where)
}

View File

@@ -0,0 +1,213 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"bytes"
"io"
"strconv"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
)
const errorPrefix = "jsontext: "
type ioError struct {
action string // either "read" or "write"
err error
}
func (e *ioError) Error() string {
return errorPrefix + e.action + " error: " + e.err.Error()
}
func (e *ioError) Unwrap() error {
return e.err
}
type numError struct {
accessor string // either "Int", "Uint", or "Float"
value string // e.g., "1e1000"
err error // either [strconv.ErrSyntax] or [strconv.ErrRange]
}
func (e *numError) Error() string {
return "jsontext.Token(" + e.value + ")." + e.accessor + " error: " + e.err.Error()
}
func (e *numError) Unwrap() error {
return e.err
}
// SyntacticError is a description of a syntactic error that occurred when
// encoding or decoding JSON according to the grammar.
//
// The contents of this error as produced by this package may change over time.
type SyntacticError struct {
requireKeyedLiterals
nonComparable
// ByteOffset indicates that an error occurred at or after this byte offset.
ByteOffset int64
// JSONPointer indicates that an error occurred within this JSON value
// as indicated using the JSON Pointer notation (see RFC 6901).
JSONPointer Pointer
// Err is the underlying error.
Err error
}
// wrapSyntacticError wraps an error and annotates it with a precise location
// using the provided [encoderState] or [decoderState].
// If err is an [ioError] or [io.EOF], then it is not wrapped.
//
// It takes a relative offset pos that can be resolved into
// an absolute offset using state.offsetAt.
//
// It takes a where that specifies how the JSON pointer is derived.
// If the underlying error is a [pointerSuffixError],
// then the suffix is appended to the derived pointer.
func wrapSyntacticError(state interface {
options() *jsonopts.Struct
offsetAt(pos int) int64
AppendStackPointer(b []byte, where int) []byte
}, err error, pos, where int) error {
if _, ok := err.(*ioError); err == io.EOF || ok {
return err
}
offset := state.offsetAt(pos)
ptr := state.AppendStackPointer(nil, where)
if serr, ok := err.(*pointerSuffixError); ok {
ptr = serr.appendPointer(ptr)
err = serr.error
}
if d, ok := state.(*decoderState); ok && err == errMismatchDelim {
where := "at start of value"
if len(d.Tokens.Stack) > 0 && d.Tokens.Last.Length() > 0 {
switch {
case d.Tokens.Last.isArray():
where = "after array element (expecting ',' or ']')"
ptr = []byte(Pointer(ptr).Parent()) // problem is with parent array
case d.Tokens.Last.isObject():
where = "after object value (expecting ',' or '}')"
ptr = []byte(Pointer(ptr).Parent()) // problem is with parent object
}
}
err = jsonwire.NewInvalidCharacterError(d.buf[pos:], where)
}
if state.options().Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) && err != io.ErrUnexpectedEOF {
// The v1 error offset more consistently reported
// the number of bytes read that lead to the error.
// This implies that the offset is after the invalid text.
if werr, ok := err.(*jsonwire.InvalidTextError); ok && werr != nil {
offset += int64(len(werr.What))
} else {
// This implies a structural error (e.g., parsing a JSON number
// when the grammar is expecting a JSON string for the object name).
// In such a case, simply increment by one, since the first byte
// is usually sufficient to indicate that the incoming token
// cannot possibly be correct.
offset++
}
}
return &SyntacticError{ByteOffset: offset, JSONPointer: Pointer(ptr), Err: err}
}
func (e *SyntacticError) Error() string {
pointer := e.JSONPointer
offset := e.ByteOffset
b := []byte(errorPrefix)
if e.Err != nil {
b = append(b, e.Err.Error()...)
if e.Err == ErrDuplicateName {
b = strconv.AppendQuote(append(b, ' '), pointer.LastToken())
pointer = pointer.Parent()
offset = 0 // not useful to print offset for duplicate names
}
} else {
b = append(b, "syntactic error"...)
}
if pointer != "" {
b = strconv.AppendQuote(append(b, " within "...), jsonwire.TruncatePointer(string(pointer), 100))
}
if offset > 0 {
b = strconv.AppendInt(append(b, " after offset "...), offset, 10)
}
return string(b)
}
func (e *SyntacticError) Unwrap() error {
return e.Err
}
// pointerSuffixError represents a JSON pointer suffix to be appended
// to [SyntacticError.JSONPointer]. It is an internal error type
// used within this package and does not appear in the public API.
//
// This type is primarily used to annotate errors in Encoder.WriteValue
// and Decoder.ReadValue with precise positions.
// At the time WriteValue or ReadValue is called, a JSON pointer to the
// upcoming value can be constructed using the Encoder/Decoder state.
// However, tracking pointers within values during normal operation
// would incur a performance penalty in the error-free case.
//
// To provide precise error locations without this overhead,
// the error is wrapped with object names or array indices
// as the call stack is popped when an error occurs.
// Since this happens in reverse order, pointerSuffixError holds
// the pointer in reverse and is only later reversed when appending to
// the pointer prefix.
//
// For example, if the encoder is at "/alpha/bravo/charlie"
// and an error occurs in WriteValue at "/xray/yankee/zulu", then
// the final pointer should be "/alpha/bravo/charlie/xray/yankee/zulu".
//
// As pointerSuffixError is populated during the error return path,
// it first contains "/zulu", then "/zulu/yankee",
// and finally "/zulu/yankee/xray".
// These tokens are reversed and concatenated to "/alpha/bravo/charlie"
// to form the full pointer.
type pointerSuffixError struct {
error
// reversePointer is a JSON pointer, but with each token in reverse order.
reversePointer []byte
}
// wrapWithObjectName wraps err with a JSON object name access,
// which must be a valid quoted JSON string.
func wrapWithObjectName(err error, quotedName []byte) error {
serr, _ := err.(*pointerSuffixError)
if serr == nil {
serr = &pointerSuffixError{error: err}
}
name := jsonwire.UnquoteMayCopy(quotedName, false)
serr.reversePointer = appendEscapePointerName(append(serr.reversePointer, '/'), name)
return serr
}
// wrapWithArrayIndex wraps err with a JSON array index access.
func wrapWithArrayIndex(err error, index int64) error {
serr, _ := err.(*pointerSuffixError)
if serr == nil {
serr = &pointerSuffixError{error: err}
}
serr.reversePointer = strconv.AppendUint(append(serr.reversePointer, '/'), uint64(index), 10)
return serr
}
// appendPointer appends the path encoded in e to the end of pointer.
func (e *pointerSuffixError) appendPointer(pointer []byte) []byte {
// Copy each token in reversePointer to the end of pointer in reverse order.
// Double reversal means that the appended suffix is now in forward order.
bi, bo := e.reversePointer, pointer
for len(bi) > 0 {
i := bytes.LastIndexByte(bi, '/')
bi, bo = bi[:i], append(bo, bi[i:]...)
}
return bo
}

View File

@@ -0,0 +1,91 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"io"
"github.com/go-json-experiment/json/internal"
)
// TODO(https://go.dev/issue/73435): Remove the Internal symbol.
//
// The Go language lacks a 3rd category of visibility where
// certain symbols can only be referenced from within the same module.
// One solution to this is to put the entirety of a package with
// both public and module-only symbols exposed as an internal package.
// A separate, public package can re-export all of the public symbols
// via type aliases and thus ensuring module-only symbols cannot be
// referenced by the end-user. While this works, it unfortunately
// leads to a poor user experience since the Go pkgsite is unable to
// forward the documentation for symbols like methods and fields.
// We need to improve the pkgsite experience before we can delete
// the Internal symbol.
// Internal is for internal use only.
// This is exempt from the Go compatibility agreement.
var Internal exporter
type exporter struct{}
// Export exposes internal functionality from "jsontext" to "json".
// This cannot be dynamically called by other packages since
// they cannot obtain a reference to the internal.AllowInternalUse value.
func (exporter) Export(p *internal.NotForPublicUse) export {
if p != &internal.AllowInternalUse {
panic("unauthorized call to Export")
}
return export{}
}
// The export type exposes functionality to packages with visibility to
// the internal.AllowInternalUse variable. The "json" package uses this
// to modify low-level state in the Encoder and Decoder types.
// It mutates the state directly instead of calling ReadToken or WriteToken
// since this is more performant. The public APIs need to track state to ensure
// that users are constructing a valid JSON value, but the "json" implementation
// guarantees that it emits valid JSON by the structure of the code itself.
type export struct{}
// Encoder returns a pointer to the underlying encoderState.
func (export) Encoder(e *Encoder) *encoderState { return &e.s }
// Decoder returns a pointer to the underlying decoderState.
func (export) Decoder(d *Decoder) *decoderState { return &d.s }
func (export) GetBufferedEncoder(o ...Options) *Encoder {
return getBufferedEncoder(o...)
}
func (export) PutBufferedEncoder(e *Encoder) {
putBufferedEncoder(e)
}
func (export) GetStreamingEncoder(w io.Writer, o ...Options) *Encoder {
return getStreamingEncoder(w, o...)
}
func (export) PutStreamingEncoder(e *Encoder) {
putStreamingEncoder(e)
}
func (export) GetBufferedDecoder(b []byte, o ...Options) *Decoder {
return getBufferedDecoder(b, o...)
}
func (export) PutBufferedDecoder(d *Decoder) {
putBufferedDecoder(d)
}
func (export) GetStreamingDecoder(r io.Reader, o ...Options) *Decoder {
return getStreamingDecoder(r, o...)
}
func (export) PutStreamingDecoder(d *Decoder) {
putStreamingDecoder(d)
}
func (export) IsIOError(err error) bool {
_, ok := err.(*ioError)
return ok
}

View File

@@ -0,0 +1,304 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"strings"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// Options configures [NewEncoder], [Encoder.Reset], [NewDecoder],
// and [Decoder.Reset] with specific features.
// Each function takes in a variadic list of options, where properties
// set in later options override the value of previously set properties.
//
// There is a single Options type, which is used with both encoding and decoding.
// Some options affect both operations, while others only affect one operation:
//
// - [AllowDuplicateNames] affects encoding and decoding
// - [AllowInvalidUTF8] affects encoding and decoding
// - [EscapeForHTML] affects encoding only
// - [EscapeForJS] affects encoding only
// - [PreserveRawStrings] affects encoding only
// - [CanonicalizeRawInts] affects encoding only
// - [CanonicalizeRawFloats] affects encoding only
// - [ReorderRawObjects] affects encoding only
// - [SpaceAfterColon] affects encoding only
// - [SpaceAfterComma] affects encoding only
// - [Multiline] affects encoding only
// - [WithIndent] affects encoding only
// - [WithIndentPrefix] affects encoding only
//
// Options that do not affect a particular operation are ignored.
//
// The Options type is identical to [encoding/json.Options] and
// [encoding/json/v2.Options]. Options from the other packages may
// be passed to functionality in this package, but are ignored.
// Options from this package may be used with the other packages.
type Options = jsonopts.Options
// AllowDuplicateNames specifies that JSON objects may contain
// duplicate member names. Disabling the duplicate name check may provide
// performance benefits, but breaks compliance with RFC 7493, section 2.3.
// The input or output will still be compliant with RFC 8259,
// which leaves the handling of duplicate names as unspecified behavior.
//
// This affects either encoding or decoding.
func AllowDuplicateNames(v bool) Options {
if v {
return jsonflags.AllowDuplicateNames | 1
} else {
return jsonflags.AllowDuplicateNames | 0
}
}
// AllowInvalidUTF8 specifies that JSON strings may contain invalid UTF-8,
// which will be mangled as the Unicode replacement character, U+FFFD.
// This causes the encoder or decoder to break compliance with
// RFC 7493, section 2.1, and RFC 8259, section 8.1.
//
// This affects either encoding or decoding.
func AllowInvalidUTF8(v bool) Options {
if v {
return jsonflags.AllowInvalidUTF8 | 1
} else {
return jsonflags.AllowInvalidUTF8 | 0
}
}
// EscapeForHTML specifies that '<', '>', and '&' characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u003c) so that
// the output is safe to embed within HTML.
//
// This only affects encoding and is ignored when decoding.
func EscapeForHTML(v bool) Options {
if v {
return jsonflags.EscapeForHTML | 1
} else {
return jsonflags.EscapeForHTML | 0
}
}
// EscapeForJS specifies that U+2028 and U+2029 characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u2028) so that
// the output is valid to embed within JavaScript. See RFC 8259, section 12.
//
// This only affects encoding and is ignored when decoding.
func EscapeForJS(v bool) Options {
if v {
return jsonflags.EscapeForJS | 1
} else {
return jsonflags.EscapeForJS | 0
}
}
// PreserveRawStrings specifies that when encoding a raw JSON string in a
// [Token] or [Value], pre-escaped sequences
// in a JSON string are preserved to the output.
// However, raw strings still respect [EscapeForHTML] and [EscapeForJS]
// such that the relevant characters are escaped.
// If [AllowInvalidUTF8] is enabled, bytes of invalid UTF-8
// are preserved to the output.
//
// This only affects encoding and is ignored when decoding.
func PreserveRawStrings(v bool) Options {
if v {
return jsonflags.PreserveRawStrings | 1
} else {
return jsonflags.PreserveRawStrings | 0
}
}
// CanonicalizeRawInts specifies that when encoding a raw JSON
// integer number (i.e., a number without a fraction and exponent) in a
// [Token] or [Value], the number is canonicalized
// according to RFC 8785, section 3.2.2.3. As a special case,
// the number -0 is canonicalized as 0.
//
// JSON numbers are treated as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example,
// integer values beyond ±2⁵³ will lose their precision.
// For example, 1234567890123456789 is formatted as 1234567890123456800.
//
// This only affects encoding and is ignored when decoding.
func CanonicalizeRawInts(v bool) Options {
if v {
return jsonflags.CanonicalizeRawInts | 1
} else {
return jsonflags.CanonicalizeRawInts | 0
}
}
// CanonicalizeRawFloats specifies that when encoding a raw JSON
// floating-point number (i.e., a number with a fraction or exponent) in a
// [Token] or [Value], the number is canonicalized
// according to RFC 8785, section 3.2.2.3. As a special case,
// the number -0 is canonicalized as 0.
//
// JSON numbers are treated as IEEE 754 double precision numbers.
// It is safe to canonicalize a serialized single precision number and
// parse it back as a single precision number and expect the same value.
// If a number exceeds ±1.7976931348623157e+308, which is the maximum
// finite number, then it is saturated at that value and formatted as such.
//
// This only affects encoding and is ignored when decoding.
func CanonicalizeRawFloats(v bool) Options {
if v {
return jsonflags.CanonicalizeRawFloats | 1
} else {
return jsonflags.CanonicalizeRawFloats | 0
}
}
// ReorderRawObjects specifies that when encoding a raw JSON object in a
// [Value], the object members are reordered according to
// RFC 8785, section 3.2.3.
//
// This only affects encoding and is ignored when decoding.
func ReorderRawObjects(v bool) Options {
if v {
return jsonflags.ReorderRawObjects | 1
} else {
return jsonflags.ReorderRawObjects | 0
}
}
// SpaceAfterColon specifies that the JSON output should emit a space character
// after each colon separator following a JSON object name.
// If false, then no space character appears after the colon separator.
//
// This only affects encoding and is ignored when decoding.
func SpaceAfterColon(v bool) Options {
if v {
return jsonflags.SpaceAfterColon | 1
} else {
return jsonflags.SpaceAfterColon | 0
}
}
// SpaceAfterComma specifies that the JSON output should emit a space character
// after each comma separator following a JSON object value or array element.
// If false, then no space character appears after the comma separator.
//
// This only affects encoding and is ignored when decoding.
func SpaceAfterComma(v bool) Options {
if v {
return jsonflags.SpaceAfterComma | 1
} else {
return jsonflags.SpaceAfterComma | 0
}
}
// Multiline specifies that the JSON output should expand to multiple lines,
// where every JSON object member or JSON array element appears on
// a new, indented line according to the nesting depth.
//
// If [SpaceAfterColon] is not specified, then the default is true.
// If [SpaceAfterComma] is not specified, then the default is false.
// If [WithIndent] is not specified, then the default is "\t".
//
// If set to false, then the output is a single-line,
// where the only whitespace emitted is determined by the current
// values of [SpaceAfterColon] and [SpaceAfterComma].
//
// This only affects encoding and is ignored when decoding.
func Multiline(v bool) Options {
if v {
return jsonflags.Multiline | 1
} else {
return jsonflags.Multiline | 0
}
}
// WithIndent specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix (see [WithIndentPrefix])
// followed by one or more copies of indent according to the nesting depth.
// The indent must only be composed of space or tab characters.
//
// If the intent is to emit indented output without a preference for
// the particular indent string, then use [Multiline] instead.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Multiline] being set to true.
func WithIndent(indent string) Options {
// Fast-path: Return a constant for common indents, which avoids allocating.
// These are derived from analyzing the Go module proxy on 2023-07-01.
switch indent {
case "\t":
return jsonopts.Indent("\t") // ~14k usages
case " ":
return jsonopts.Indent(" ") // ~18k usages
case " ":
return jsonopts.Indent(" ") // ~1.7k usages
case " ":
return jsonopts.Indent(" ") // ~52k usages
case " ":
return jsonopts.Indent(" ") // ~12k usages
case "":
return jsonopts.Indent("") // ~1.5k usages
}
// Otherwise, allocate for this unique value.
if s := strings.Trim(indent, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune([]byte(s)) + " in indent")
}
return jsonopts.Indent(indent)
}
// WithIndentPrefix specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix followed by one or more copies of indent
// (see [WithIndent]) according to the nesting depth.
// The prefix must only be composed of space or tab characters.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Multiline] being set to true.
func WithIndentPrefix(prefix string) Options {
if s := strings.Trim(prefix, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune([]byte(s)) + " in indent prefix")
}
return jsonopts.IndentPrefix(prefix)
}
/*
// TODO(https://go.dev/issue/56733): Implement WithByteLimit and WithDepthLimit.
// Remember to also update the "Security Considerations" section.
// WithByteLimit sets a limit on the number of bytes of input or output bytes
// that may be consumed or produced for each top-level JSON value.
// If a [Decoder] or [Encoder] method call would need to consume/produce
// more than a total of n bytes to make progress on the top-level JSON value,
// then the call will report an error.
// Whitespace before and within the top-level value are counted against the limit.
// Whitespace after a top-level value are counted against the limit
// for the next top-level value.
//
// A non-positive limit is equivalent to no limit at all.
// If unspecified, the default limit is no limit at all.
// This affects either encoding or decoding.
func WithByteLimit(n int64) Options {
return jsonopts.ByteLimit(max(n, 0))
}
// WithDepthLimit sets a limit on the maximum depth of JSON nesting
// that may be consumed or produced for each top-level JSON value.
// If a [Decoder] or [Encoder] method call would need to consume or produce
// a depth greater than n to make progress on the top-level JSON value,
// then the call will report an error.
//
// A non-positive limit is equivalent to no limit at all.
// If unspecified, the default limit is 10000.
// This affects either encoding or decoding.
func WithDepthLimit(n int) Options {
return jsonopts.DepthLimit(max(n, 0))
}
*/

View File

@@ -0,0 +1,164 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"bytes"
"io"
"math/bits"
"sync"
)
// TODO(https://go.dev/issue/47657): Use sync.PoolOf.
var (
// This owns the internal buffer since there is no io.Writer to output to.
// Since the buffer can get arbitrarily large in normal usage,
// there is statistical tracking logic to determine whether to recycle
// the internal buffer or not based on a history of utilization.
bufferedEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON before flushing it to the underlying io.Writer.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
)
// bufferStatistics is statistics to track buffer utilization.
// It is used to determine whether to recycle a buffer or not
// to avoid https://go.dev/issue/23199.
type bufferStatistics struct {
strikes int // number of times the buffer was under-utilized
prevLen int // length of previous buffer
}
func getBufferedEncoder(opts ...Options) *Encoder {
e := bufferedEncoderPool.Get().(*Encoder)
if e.s.Buf == nil {
// Round up to nearest 2ⁿ to make best use of malloc size classes.
// See runtime/sizeclasses.go on Go1.15.
// Logical OR with 63 to ensure 64 as the minimum buffer size.
n := 1 << bits.Len(uint(e.s.bufStats.prevLen|63))
e.s.Buf = make([]byte, 0, n)
}
e.s.reset(e.s.Buf[:0], nil, opts...)
return e
}
func putBufferedEncoder(e *Encoder) {
if cap(e.s.availBuffer) > 64<<10 {
e.s.availBuffer = nil // avoid pinning arbitrarily large amounts of memory
}
// Recycle large buffers only if sufficiently utilized.
// If a buffer is under-utilized enough times sequentially,
// then it is discarded, ensuring that a single large buffer
// won't be kept alive by a continuous stream of small usages.
//
// The worst case utilization is computed as:
// MIN_UTILIZATION_THRESHOLD / (1 + MAX_NUM_STRIKES)
//
// For the constants chosen below, this is (25%)/(1+4) ⇒ 5%.
// This may seem low, but it ensures a lower bound on
// the absolute worst-case utilization. Without this check,
// this would be theoretically 0%, which is infinitely worse.
//
// See https://go.dev/issue/27735.
switch {
case cap(e.s.Buf) <= 4<<10: // always recycle buffers smaller than 4KiB
e.s.bufStats.strikes = 0
case cap(e.s.Buf)/4 <= len(e.s.Buf): // at least 25% utilization
e.s.bufStats.strikes = 0
case e.s.bufStats.strikes < 4: // at most 4 strikes
e.s.bufStats.strikes++
default: // discard the buffer; too large and too often under-utilized
e.s.bufStats.strikes = 0
e.s.bufStats.prevLen = len(e.s.Buf) // heuristic for size to allocate next time
e.s.Buf = nil
}
bufferedEncoderPool.Put(e)
}
func getStreamingEncoder(w io.Writer, opts ...Options) *Encoder {
if _, ok := w.(*bytes.Buffer); ok {
e := bytesBufferEncoderPool.Get().(*Encoder)
e.s.reset(nil, w, opts...) // buffer taken from bytes.Buffer
return e
} else {
e := streamingEncoderPool.Get().(*Encoder)
e.s.reset(e.s.Buf[:0], w, opts...) // preserve existing buffer
return e
}
}
func putStreamingEncoder(e *Encoder) {
if cap(e.s.availBuffer) > 64<<10 {
e.s.availBuffer = nil // avoid pinning arbitrarily large amounts of memory
}
if _, ok := e.s.wr.(*bytes.Buffer); ok {
e.s.wr, e.s.Buf = nil, nil // avoid pinning the provided bytes.Buffer
bytesBufferEncoderPool.Put(e)
} else {
e.s.wr = nil // avoid pinning the provided io.Writer
if cap(e.s.Buf) > 64<<10 {
e.s.Buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingEncoderPool.Put(e)
}
}
var (
// This does not own the internal buffer since it is externally provided.
bufferedDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON fetched from the underlying io.Reader.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferDecoderPool = bufferedDecoderPool
)
func getBufferedDecoder(b []byte, opts ...Options) *Decoder {
d := bufferedDecoderPool.Get().(*Decoder)
d.s.reset(b, nil, opts...)
return d
}
func putBufferedDecoder(d *Decoder) {
d.s.buf = nil // avoid pinning the provided buffer
bufferedDecoderPool.Put(d)
}
func getStreamingDecoder(r io.Reader, opts ...Options) *Decoder {
if _, ok := r.(*bytes.Buffer); ok {
d := bytesBufferDecoderPool.Get().(*Decoder)
d.s.reset(nil, r, opts...) // buffer taken from bytes.Buffer
return d
} else {
d := streamingDecoderPool.Get().(*Decoder)
d.s.reset(d.s.buf[:0], r, opts...) // preserve existing buffer
return d
}
}
func putStreamingDecoder(d *Decoder) {
if _, ok := d.s.rd.(*bytes.Buffer); ok {
d.s.rd, d.s.buf = nil, nil // avoid pinning the provided bytes.Buffer
bytesBufferDecoderPool.Put(d)
} else {
d.s.rd = nil // avoid pinning the provided io.Reader
if cap(d.s.buf) > 64<<10 {
d.s.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingDecoderPool.Put(d)
}
}

View File

@@ -0,0 +1,41 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// AppendQuote appends a double-quoted JSON string literal representing src
// to dst and returns the extended buffer.
// It uses the minimal string representation per RFC 8785, section 3.2.2.2.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
// The dst must not overlap with the src.
func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
dst, err := jsonwire.AppendQuote(dst, []byte(src), &jsonflags.Flags{})
if err != nil {
err = &SyntacticError{Err: err}
}
return dst, err
}
// AppendUnquote appends the decoded interpretation of src as a
// double-quoted JSON string literal to dst and returns the extended buffer.
// The input src must be a JSON string without any surrounding whitespace.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
// Any trailing bytes after the JSON string literal results in an error.
// The dst must not overlap with the src.
func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
dst, err := jsonwire.AppendUnquote(dst, []byte(src))
if err != nil {
err = &SyntacticError{Err: err}
}
return dst, err
}

View File

@@ -0,0 +1,828 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"errors"
"iter"
"math"
"strconv"
"strings"
"unicode/utf8"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// ErrDuplicateName indicates that a JSON token could not be
// encoded or decoded because it results in a duplicate JSON object name.
// This error is directly wrapped within a [SyntacticError] when produced.
//
// The name of a duplicate JSON object member can be extracted as:
//
// err := ...
// serr, ok := errors.AsType[*jsontext.SyntacticError](err)
// if ok && serr.Err == jsontext.ErrDuplicateName {
// ptr := serr.JSONPointer // JSON pointer to duplicate name
// name := ptr.LastToken() // duplicate name itself
// ...
// }
//
// This error is only returned if [AllowDuplicateNames] is false.
var ErrDuplicateName = errors.New("duplicate object member name")
// ErrNonStringName indicates that a JSON token could not be
// encoded or decoded because it is not a string,
// as required for JSON object names according to RFC 8259, section 4.
// This error is directly wrapped within a [SyntacticError] when produced.
var ErrNonStringName = errors.New("object member name must be a string")
var (
errMissingValue = errors.New("missing value after object name")
errMismatchDelim = errors.New("mismatching structural token for object or array")
errMaxDepth = errors.New("exceeded max depth")
errInvalidNamespace = errors.New("object namespace is in an invalid state")
)
// Per RFC 8259, section 9, implementations may enforce a maximum depth.
// Such a limit is necessary to prevent stack overflows.
const maxNestingDepth = 10000
type state struct {
// Tokens validates whether the next token kind is valid.
Tokens stateMachine
// Names is a stack of object names.
Names objectNameStack
// Namespaces is a stack of object namespaces.
// For performance reasons, Encoder or Decoder may not update this
// if Marshal or Unmarshal is able to track names in a more efficient way.
// See makeMapArshaler and makeStructArshaler.
// Not used if AllowDuplicateNames is true.
Namespaces objectNamespaceStack
}
// needObjectValue reports whether the next token should be an object value.
// This method is used by [wrapSyntacticError].
func (s *state) needObjectValue() bool {
return s.Tokens.Last.needObjectValue()
}
func (s *state) reset() {
s.Tokens.reset()
s.Names.reset()
s.Namespaces.reset()
}
// Pointer is a JSON Pointer (RFC 6901) that references a particular JSON value
// relative to the root of the top-level JSON value.
//
// A Pointer is a slash-separated list of tokens, where each token is
// either a JSON object name or an index to a JSON array element
// encoded as a base-10 integer value.
// It is impossible to distinguish between an array index and an object name
// (that happens to be a base-10 encoded integer) without also knowing
// the structure of the top-level JSON value that the pointer refers to.
//
// There is exactly one representation of a pointer to a particular value,
// so comparability of Pointer values is equivalent to checking whether
// they both point to the exact same value.
type Pointer string
// IsValid reports whether p is a valid JSON Pointer according to RFC 6901.
// Note that the concatenation of two valid pointers produces a valid pointer.
func (p Pointer) IsValid() bool {
for i, r := range p {
switch {
case r == '~' && (i+1 == len(p) || (p[i+1] != '0' && p[i+1] != '1')):
return false // invalid escape
case r == '\ufffd' && !strings.HasPrefix(string(p[i:]), "\ufffd"):
return false // invalid UTF-8
}
}
return len(p) == 0 || p[0] == '/'
}
// Contains reports whether the JSON value that p points to
// is equal to or contains the JSON value that pc points to.
func (p Pointer) Contains(pc Pointer) bool {
// Invariant: len(p) <= len(pc) if p.Contains(pc)
suffix, ok := strings.CutPrefix(string(pc), string(p))
return ok && (suffix == "" || suffix[0] == '/')
}
// Parent strips off the last token and returns the remaining pointer.
// The parent of an empty p is an empty string.
func (p Pointer) Parent() Pointer {
return p[:max(strings.LastIndexByte(string(p), '/'), 0)]
}
// LastToken returns the last token in the pointer.
// The last token of an empty p is an empty string.
func (p Pointer) LastToken() string {
last := p[max(strings.LastIndexByte(string(p), '/'), 0):]
return unescapePointerToken(strings.TrimPrefix(string(last), "/"))
}
// AppendToken appends a token to the end of p and returns the full pointer.
func (p Pointer) AppendToken(tok string) Pointer {
return Pointer(appendEscapePointerName([]byte(p+"/"), []byte(tok)))
}
// TODO: Add Pointer.AppendTokens,
// but should this take in a ...string or an iter.Seq[string]?
// Tokens returns an iterator over the reference tokens in the JSON pointer,
// starting from the first token until the last token (unless stopped early).
func (p Pointer) Tokens() iter.Seq[string] {
return func(yield func(string) bool) {
for len(p) > 0 {
p = Pointer(strings.TrimPrefix(string(p), "/"))
i := min(uint(strings.IndexByte(string(p), '/')), uint(len(p)))
if !yield(unescapePointerToken(string(p)[:i])) {
return
}
p = p[i:]
}
}
}
func unescapePointerToken(token string) string {
if strings.Contains(token, "~") {
// Per RFC 6901, section 3, unescape '~' and '/' characters.
token = strings.ReplaceAll(token, "~1", "/")
token = strings.ReplaceAll(token, "~0", "~")
}
return token
}
// appendStackPointer appends a JSON Pointer (RFC 6901) to the current value.
//
// - If where is -1, then it points to the previously processed token.
//
// - If where is 0, then it points to the parent JSON object or array,
// or an object member if in-between an object member key and value.
// This is useful when the position is ambiguous whether
// we are interested in the previous or next token, or
// when we are uncertain whether the next token
// continues or terminates the current object or array.
//
// - If where is +1, then it points to the next expected value,
// assuming that it continues the current JSON object or array.
// As a special case, if the next token is a JSON object name,
// then it points to the parent JSON object.
//
// Invariant: Must call s.names.copyQuotedBuffer beforehand.
func (s state) appendStackPointer(b []byte, where int) []byte {
var objectDepth int
for i := 1; i < s.Tokens.Depth(); i++ {
e := s.Tokens.index(i)
arrayDelta := -1 // by default point to previous array element
if isLast := i == s.Tokens.Depth()-1; isLast {
switch {
case where < 0 && e.Length() == 0 || where == 0 && !e.needObjectValue() || where > 0 && e.NeedObjectName():
return b
case where > 0 && e.isArray():
arrayDelta = 0 // point to next array element
}
}
switch {
case e.isObject():
b = appendEscapePointerName(append(b, '/'), s.Names.getUnquoted(objectDepth))
objectDepth++
case e.isArray():
b = strconv.AppendUint(append(b, '/'), uint64(e.Length()+int64(arrayDelta)), 10)
}
}
return b
}
func appendEscapePointerName(b, name []byte) []byte {
for _, r := range string(name) {
// Per RFC 6901, section 3, escape '~' and '/' characters.
switch r {
case '~':
b = append(b, "~0"...)
case '/':
b = append(b, "~1"...)
default:
b = utf8.AppendRune(b, r)
}
}
return b
}
// stateMachine is a push-down automaton that validates whether
// a sequence of tokens is valid or not according to the JSON grammar.
// It is useful for both encoding and decoding.
//
// It is a stack where each entry represents a nested JSON object or array.
// The stack has a minimum depth of 1 where the first level is a
// virtual JSON array to handle a stream of top-level JSON values.
// The top-level virtual JSON array is special in that it doesn't require commas
// between each JSON value.
//
// For performance, most methods are carefully written to be inlinable.
// The zero value is a valid state machine ready for use.
type stateMachine struct {
Stack []stateEntry
Last stateEntry
}
// reset resets the state machine.
// The machine always starts with a minimum depth of 1.
func (m *stateMachine) reset() {
m.Stack = m.Stack[:0]
if cap(m.Stack) > 1<<10 {
m.Stack = nil
}
m.Last = stateTypeArray
}
// Depth is the current nested depth of JSON objects and arrays.
// It is one-indexed (i.e., top-level values have a depth of 1).
func (m stateMachine) Depth() int {
return len(m.Stack) + 1
}
// index returns a reference to the ith entry.
// It is only valid until the next push method call.
func (m *stateMachine) index(i int) *stateEntry {
if i == len(m.Stack) {
return &m.Last
}
return &m.Stack[i]
}
// DepthLength reports the current nested depth and
// the length of the last JSON object or array.
func (m stateMachine) DepthLength() (int, int64) {
return m.Depth(), m.Last.Length()
}
// appendLiteral appends a JSON literal as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendLiteral() error {
switch {
case m.Last.NeedObjectName():
return ErrNonStringName
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last.Increment()
return nil
}
}
// appendString appends a JSON string as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendString() error {
switch {
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last.Increment()
return nil
}
}
// appendNumber appends a JSON number as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendNumber() error {
return m.appendLiteral()
}
// pushObject appends a JSON begin object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushObject() error {
switch {
case m.Last.NeedObjectName():
return ErrNonStringName
case !m.Last.isValidNamespace():
return errInvalidNamespace
case len(m.Stack) == maxNestingDepth:
return errMaxDepth
default:
m.Last.Increment()
m.Stack = append(m.Stack, m.Last)
m.Last = stateTypeObject
return nil
}
}
// popObject appends a JSON end object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popObject() error {
switch {
case !m.Last.isObject():
return errMismatchDelim
case m.Last.needObjectValue():
return errMissingValue
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last = m.Stack[len(m.Stack)-1]
m.Stack = m.Stack[:len(m.Stack)-1]
return nil
}
}
// pushArray appends a JSON begin array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushArray() error {
switch {
case m.Last.NeedObjectName():
return ErrNonStringName
case !m.Last.isValidNamespace():
return errInvalidNamespace
case len(m.Stack) == maxNestingDepth:
return errMaxDepth
default:
m.Last.Increment()
m.Stack = append(m.Stack, m.Last)
m.Last = stateTypeArray
return nil
}
}
// popArray appends a JSON end array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popArray() error {
switch {
case !m.Last.isArray() || len(m.Stack) == 0: // forbid popping top-level virtual JSON array
return errMismatchDelim
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last = m.Stack[len(m.Stack)-1]
m.Stack = m.Stack[:len(m.Stack)-1]
return nil
}
}
// NeedIndent reports whether indent whitespace should be injected.
// A zero value means that no whitespace should be injected.
// A positive value means '\n', indentPrefix, and (n-1) copies of indentBody
// should be appended to the output immediately before the next token.
func (m stateMachine) NeedIndent(next Kind) (n int) {
willEnd := next == '}' || next == ']'
switch {
case m.Depth() == 1:
return 0 // top-level values are never indented
case m.Last.Length() == 0 && willEnd:
return 0 // an empty object or array is never indented
case m.Last.Length() == 0 || m.Last.needImplicitComma(next):
return m.Depth()
case willEnd:
return m.Depth() - 1
default:
return 0
}
}
// MayAppendDelim appends a colon or comma that may precede the next token.
func (m stateMachine) MayAppendDelim(b []byte, next Kind) []byte {
switch {
case m.Last.needImplicitColon():
return append(b, ':')
case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values
return append(b, ',')
default:
return b
}
}
// needDelim reports whether a colon or comma token should be implicitly emitted
// before the next token of the specified kind.
// A zero value means no delimiter should be emitted.
func (m stateMachine) needDelim(next Kind) (delim byte) {
switch {
case m.Last.needImplicitColon():
return ':'
case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values
return ','
default:
return 0
}
}
// InvalidateDisabledNamespaces marks all disabled namespaces as invalid.
//
// For efficiency, Marshal and Unmarshal may disable namespaces since there are
// more efficient ways to track duplicate names. However, if an error occurs,
// the namespaces in Encoder or Decoder will be left in an inconsistent state.
// Mark the namespaces as invalid so that future method calls on
// Encoder or Decoder will return an error.
func (m *stateMachine) InvalidateDisabledNamespaces() {
for i := range m.Depth() {
e := m.index(i)
if !e.isActiveNamespace() {
e.invalidateNamespace()
}
}
}
// stateEntry encodes several artifacts within a single unsigned integer:
// - whether this represents a JSON object or array,
// - whether this object should check for duplicate names, and
// - how many elements are in this JSON object or array.
type stateEntry uint64
const (
// The type mask (1 bit) records whether this is a JSON object or array.
stateTypeMask stateEntry = 0x8000_0000_0000_0000
stateTypeObject stateEntry = 0x8000_0000_0000_0000
stateTypeArray stateEntry = 0x0000_0000_0000_0000
// The name check mask (2 bits) records whether to update
// the namespaces for the current JSON object and
// whether the namespace is valid.
stateNamespaceMask stateEntry = 0x6000_0000_0000_0000
stateDisableNamespace stateEntry = 0x4000_0000_0000_0000
stateInvalidNamespace stateEntry = 0x2000_0000_0000_0000
// The count mask (61 bits) records the number of elements.
stateCountMask stateEntry = 0x1fff_ffff_ffff_ffff
stateCountLSBMask stateEntry = 0x0000_0000_0000_0001
stateCountOdd stateEntry = 0x0000_0000_0000_0001
stateCountEven stateEntry = 0x0000_0000_0000_0000
)
// Length reports the number of elements in the JSON object or array.
// Each name and value in an object entry is treated as a separate element.
func (e stateEntry) Length() int64 {
return int64(e & stateCountMask)
}
// isObject reports whether this is a JSON object.
func (e stateEntry) isObject() bool {
return e&stateTypeMask == stateTypeObject
}
// isArray reports whether this is a JSON array.
func (e stateEntry) isArray() bool {
return e&stateTypeMask == stateTypeArray
}
// NeedObjectName reports whether the next token must be a JSON string,
// which is necessary for JSON object names.
func (e stateEntry) NeedObjectName() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountEven
}
// needImplicitColon reports whether a colon should occur next,
// which always occurs after JSON object names.
func (e stateEntry) needImplicitColon() bool {
return e.needObjectValue()
}
// needObjectValue reports whether the next token must be a JSON value,
// which is necessary after every JSON object name.
func (e stateEntry) needObjectValue() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountOdd
}
// needImplicitComma reports whether a comma should occur next,
// which always occurs after a value in a JSON object or array
// before the next value (or name).
func (e stateEntry) needImplicitComma(next Kind) bool {
return !e.needObjectValue() && e.Length() > 0 && next != '}' && next != ']'
}
// Increment increments the number of elements for the current object or array.
// This assumes that overflow won't practically be an issue since
// 1<<bits.OnesCount(stateCountMask) is sufficiently large.
func (e *stateEntry) Increment() {
(*e)++
}
// decrement decrements the number of elements for the current object or array.
// It is the caller's responsibility to ensure that e.length > 0.
func (e *stateEntry) decrement() {
(*e)--
}
// DisableNamespace disables the JSON object namespace such that the
// Encoder or Decoder no longer updates the namespace.
func (e *stateEntry) DisableNamespace() {
*e |= stateDisableNamespace
}
// isActiveNamespace reports whether the JSON object namespace is actively
// being updated and used for duplicate name checks.
func (e stateEntry) isActiveNamespace() bool {
return e&(stateDisableNamespace) == 0
}
// invalidateNamespace marks the JSON object namespace as being invalid.
func (e *stateEntry) invalidateNamespace() {
*e |= stateInvalidNamespace
}
// isValidNamespace reports whether the JSON object namespace is valid.
func (e stateEntry) isValidNamespace() bool {
return e&(stateInvalidNamespace) == 0
}
// objectNameStack is a stack of names when descending into a JSON object.
// In contrast to objectNamespaceStack, this only has to remember a single name
// per JSON object.
//
// This data structure may contain offsets to encodeBuffer or decodeBuffer.
// It violates clean abstraction of layers, but is significantly more efficient.
// This ensures that popping and pushing in the common case is a trivial
// push/pop of an offset integer.
//
// The zero value is an empty names stack ready for use.
type objectNameStack struct {
// offsets is a stack of offsets for each name.
// A non-negative offset is the ending offset into the local names buffer.
// A negative offset is the bit-wise inverse of a starting offset into
// a remote buffer (e.g., encodeBuffer or decodeBuffer).
// A math.MinInt offset at the end implies that the last object is empty.
// Invariant: Positive offsets always occur before negative offsets.
offsets []int
// unquotedNames is a back-to-back concatenation of names.
unquotedNames []byte
}
func (ns *objectNameStack) reset() {
ns.offsets = ns.offsets[:0]
ns.unquotedNames = ns.unquotedNames[:0]
if cap(ns.offsets) > 1<<6 {
ns.offsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.unquotedNames) > 1<<10 {
ns.unquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
func (ns *objectNameStack) length() int {
return len(ns.offsets)
}
// getUnquoted retrieves the ith unquoted name in the stack.
// It returns an empty string if the last object is empty.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) getUnquoted(i int) []byte {
ns.ensureCopiedBuffer()
if i == 0 {
return ns.unquotedNames[:ns.offsets[0]]
} else {
return ns.unquotedNames[ns.offsets[i-1]:ns.offsets[i]]
}
}
// invalidOffset indicates that the last JSON object currently has no name.
const invalidOffset = math.MinInt
// push descends into a nested JSON object.
func (ns *objectNameStack) push() {
ns.offsets = append(ns.offsets, invalidOffset)
}
// ReplaceLastQuotedOffset replaces the last name with the starting offset
// to the quoted name in some remote buffer. All offsets provided must be
// relative to the same buffer until copyQuotedBuffer is called.
func (ns *objectNameStack) ReplaceLastQuotedOffset(i int) {
// Use bit-wise inversion instead of naive multiplication by -1 to avoid
// ambiguity regarding zero (which is a valid offset into the names field).
// Bit-wise inversion is mathematically equivalent to -i-1,
// such that 0 becomes -1, 1 becomes -2, and so forth.
// This ensures that remote offsets are always negative.
ns.offsets[len(ns.offsets)-1] = ^i
}
// replaceLastUnquotedName replaces the last name with the provided name.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) replaceLastUnquotedName(s string) {
ns.ensureCopiedBuffer()
var startOffset int
if len(ns.offsets) > 1 {
startOffset = ns.offsets[len(ns.offsets)-2]
}
ns.unquotedNames = append(ns.unquotedNames[:startOffset], s...)
ns.offsets[len(ns.offsets)-1] = len(ns.unquotedNames)
}
// clearLast removes any name in the last JSON object.
// It is semantically equivalent to ns.push followed by ns.pop.
func (ns *objectNameStack) clearLast() {
ns.offsets[len(ns.offsets)-1] = invalidOffset
}
// pop ascends out of a nested JSON object.
func (ns *objectNameStack) pop() {
ns.offsets = ns.offsets[:len(ns.offsets)-1]
}
// copyQuotedBuffer copies names from the remote buffer into the local names
// buffer so that there are no more offset references into the remote buffer.
// This allows the remote buffer to change contents without affecting
// the names that this data structure is trying to remember.
func (ns *objectNameStack) copyQuotedBuffer(b []byte) {
// Find the first negative offset.
var i int
for i = len(ns.offsets) - 1; i >= 0 && ns.offsets[i] < 0; i-- {
continue
}
// Copy each name from the remote buffer into the local buffer.
for i = i + 1; i < len(ns.offsets); i++ {
if i == len(ns.offsets)-1 && ns.offsets[i] == invalidOffset {
if i == 0 {
ns.offsets[i] = 0
} else {
ns.offsets[i] = ns.offsets[i-1]
}
break // last JSON object had a push without any names
}
// As a form of Hyrum proofing, we write an invalid character into the
// buffer to make misuse of Decoder.ReadToken more obvious.
// We need to undo that mutation here.
quotedName := b[^ns.offsets[i]:]
if quotedName[0] == invalidateBufferByte {
quotedName[0] = '"'
}
// Append the unquoted name to the local buffer.
var startOffset int
if i > 0 {
startOffset = ns.offsets[i-1]
}
if n := jsonwire.ConsumeSimpleString(quotedName); n > 0 {
ns.unquotedNames = append(ns.unquotedNames[:startOffset], quotedName[len(`"`):n-len(`"`)]...)
} else {
ns.unquotedNames, _ = jsonwire.AppendUnquote(ns.unquotedNames[:startOffset], quotedName)
}
ns.offsets[i] = len(ns.unquotedNames)
}
}
func (ns *objectNameStack) ensureCopiedBuffer() {
if len(ns.offsets) > 0 && ns.offsets[len(ns.offsets)-1] < 0 {
panic("BUG: copyQuotedBuffer not called beforehand")
}
}
// objectNamespaceStack is a stack of object namespaces.
// This data structure assists in detecting duplicate names.
type objectNamespaceStack []objectNamespace
// reset resets the object namespace stack.
func (nss *objectNamespaceStack) reset() {
if cap(*nss) > 1<<10 {
*nss = nil
}
*nss = (*nss)[:0]
}
// push starts a new namespace for a nested JSON object.
func (nss *objectNamespaceStack) push() {
if cap(*nss) > len(*nss) {
*nss = (*nss)[:len(*nss)+1]
nss.Last().reset()
} else {
*nss = append(*nss, objectNamespace{})
}
}
// Last returns a pointer to the last JSON object namespace.
func (nss objectNamespaceStack) Last() *objectNamespace {
return &nss[len(nss)-1]
}
// pop terminates the namespace for a nested JSON object.
func (nss *objectNamespaceStack) pop() {
*nss = (*nss)[:len(*nss)-1]
}
// objectNamespace is the namespace for a JSON object.
// In contrast to objectNameStack, this needs to remember all names
// per JSON object.
//
// The zero value is an empty namespace ready for use.
type objectNamespace struct {
// It relies on a linear search over all the names before switching
// to use a Go map for direct lookup.
// endOffsets is a list of offsets to the end of each name in buffers.
// The length of offsets is the number of names in the namespace.
endOffsets []uint
// allUnquotedNames is a back-to-back concatenation of every name in the namespace.
allUnquotedNames []byte
// mapNames is a Go map containing every name in the namespace.
// Only valid if non-nil.
mapNames map[string]struct{}
}
// reset resets the namespace to be empty.
func (ns *objectNamespace) reset() {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
ns.mapNames = nil
if cap(ns.endOffsets) > 1<<6 {
ns.endOffsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.allUnquotedNames) > 1<<10 {
ns.allUnquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
// length reports the number of names in the namespace.
func (ns *objectNamespace) length() int {
return len(ns.endOffsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
func (ns *objectNamespace) getUnquoted(i int) []byte {
if i == 0 {
return ns.allUnquotedNames[:ns.endOffsets[0]]
} else {
return ns.allUnquotedNames[ns.endOffsets[i-1]:ns.endOffsets[i]]
}
}
// lastUnquoted retrieves the last name in the namespace.
func (ns *objectNamespace) lastUnquoted() []byte {
return ns.getUnquoted(ns.length() - 1)
}
// insertQuoted inserts a name and reports whether it was inserted,
// which only occurs if name is not already in the namespace.
// The provided name must be a valid JSON string.
func (ns *objectNamespace) insertQuoted(name []byte, isVerbatim bool) bool {
if isVerbatim {
name = name[len(`"`) : len(name)-len(`"`)]
}
return ns.insert(name, !isVerbatim)
}
func (ns *objectNamespace) InsertUnquoted(name []byte) bool {
return ns.insert(name, false)
}
func (ns *objectNamespace) insert(name []byte, quoted bool) bool {
var allNames []byte
if quoted {
allNames, _ = jsonwire.AppendUnquote(ns.allUnquotedNames, name)
} else {
allNames = append(ns.allUnquotedNames, name...)
}
name = allNames[len(ns.allUnquotedNames):]
// Switch to a map if the buffer is too large for linear search.
// This does not add the current name to the map.
if ns.mapNames == nil && (ns.length() > 64 || len(ns.allUnquotedNames) > 1024) {
ns.mapNames = make(map[string]struct{})
var startOffset uint
for _, endOffset := range ns.endOffsets {
name := ns.allUnquotedNames[startOffset:endOffset]
ns.mapNames[string(name)] = struct{}{} // allocates a new string
startOffset = endOffset
}
}
if ns.mapNames == nil {
// Perform linear search over the buffer to find matching names.
// It provides O(n) lookup, but does not require any allocations.
var startOffset uint
for _, endOffset := range ns.endOffsets {
if string(ns.allUnquotedNames[startOffset:endOffset]) == string(name) {
return false
}
startOffset = endOffset
}
} else {
// Use the map if it is populated.
// It provides O(1) lookup, but requires a string allocation per name.
if _, ok := ns.mapNames[string(name)]; ok {
return false
}
ns.mapNames[string(name)] = struct{}{} // allocates a new string
}
ns.allUnquotedNames = allNames
ns.endOffsets = append(ns.endOffsets, uint(len(ns.allUnquotedNames)))
return true
}
// removeLast removes the last name in the namespace.
func (ns *objectNamespace) removeLast() {
if ns.mapNames != nil {
delete(ns.mapNames, string(ns.lastUnquoted()))
}
if ns.length()-1 == 0 {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
} else {
ns.endOffsets = ns.endOffsets[:ns.length()-1]
ns.allUnquotedNames = ns.allUnquotedNames[:ns.endOffsets[ns.length()-1]]
}
}

View File

@@ -0,0 +1,697 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"bytes"
"errors"
"math"
"strconv"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// NOTE: Token is analogous to v1 json.Token.
const (
maxInt64 = math.MaxInt64
minInt64 = math.MinInt64
maxUint64 = math.MaxUint64
minUint64 = 0 // for consistency and readability purposes
invalidTokenPanic = "invalid jsontext.Token; it has been voided by a subsequent json.Decoder call"
)
var errInvalidToken = errors.New("invalid jsontext.Token")
// Token represents a lexical JSON token, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a begin or end delimiter for a JSON object (i.e., { or } )
// - a begin or end delimiter for a JSON array (i.e., [ or ] )
//
// A Token cannot represent entire array or object values, while a [Value] can.
// There is no Token to represent commas and colons since
// these structural tokens can be inferred from the surrounding context.
//
// A Token stores data in one of two forms:
//
// - As raw JSON text: backed by the internal buffer of the [Decoder]
// and only ever produced by [Decoder.ReadToken].
// Such a token is only valid until the next call to any method on that
// [Decoder] (e.g., [Decoder.PeekKind], [Decoder.ReadToken],
// [Decoder.ReadValue], or [Decoder.SkipValue]).
// Call [Token.Clone] to copy the raw text into an independent allocation
// that persists beyond subsequent [Decoder] calls.
//
// - As a typed Go value: a self-contained representation produced by
// the constructor functions (e.g., [String], [Int], [Uint], [Float]).
// Such tokens are valid indefinitely and do not need to be cloned.
type Token struct {
nonComparable
// Tokens can exist in either a "raw" or an "exact" form.
// Tokens produced by the Decoder are in the "raw" form.
// Tokens returned by constructors are usually in the "exact" form.
// The Encoder accepts Tokens in either the "raw" or "exact" form.
//
// The following chart shows the possible values for each Token type:
// ╔══════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠══════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float32) ║ nil │ "F" │ non-zero ║
// ║ number (float64) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚══════════════════╩════════════╧════════════╧════════════╝
//
// Notes:
// - For tokens stored in "raw" form, the num field contains the
// absolute offset determined by raw.previousOffsetStart().
// The buffer itself is stored in raw.previousBuffer().
// - JSON literals and structural characters are always in the "raw" form.
// - JSON strings and numbers can be in either "raw" or "exact" forms.
// - The exact zero value of JSON strings and numbers in the "exact" forms
// have ambiguous representation. Thus, they are always represented
// in the "raw" form.
// raw contains a reference to the raw decode buffer.
// If non-nil, then its value takes precedence over str and num.
// It is only valid if num == raw.previousOffsetStart().
raw *decodeBuffer
// str is the unescaped JSON string if num is zero.
// Otherwise, it is "F", "f", "i", or "u" if num should be interpreted
// as a float32, float64, int64, or uint64, respectively.
str string
// num is a float32, float64, int64, or uint64 stored as a uint64 value.
// For floating-point values, it stores the raw IEEE-754 bit-pattern.
// It is non-zero for any JSON number in the "exact" form.
num uint64
}
// TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
var (
Null Token = rawToken("null")
False Token = rawToken("false")
True Token = rawToken("true")
BeginObject Token = rawToken("{")
EndObject Token = rawToken("}")
BeginArray Token = rawToken("[")
EndArray Token = rawToken("]")
zeroString Token = rawToken(`""`)
zeroNumber Token = rawToken(`0`)
nanString Token = String("NaN")
pinfString Token = String("Infinity")
ninfString Token = String("-Infinity")
)
func rawToken(s string) Token {
return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
}
// Bool constructs a Token representing a JSON boolean.
func Bool(b bool) Token {
if b {
return True
}
return False
}
// String constructs a Token representing a JSON string.
// The provided string should contain valid UTF-8, otherwise invalid characters
// may be mangled as the Unicode replacement character.
func String(s string) Token {
if len(s) == 0 {
return zeroString
}
return Token{str: s}
}
// Float32 constructs a Token representing a JSON number as
// a 32-bit floating-point number formatted according to
// ECMA-262, 10th edition, section 7.1.12.1,
// with the exception that -0 is still formatted as -0.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers. Use of 32-bit precision should
// only be used if the corresponding decoder knows that
// this JSON number token is expected to only have 32-bit precision.
// For all other situations, prefer using the [Float] constructor instead.
func Float32(n float32) Token {
if n != 0 && !math.IsNaN(float64(n)) && !math.IsInf(float64(n), 0) {
return Token{str: "F", num: uint64(math.Float32bits(n))}
}
return Float(float64(n)) // handles ±0, NaN, and ±Inf
}
// Float constructs a Token representing a JSON number as
// a 64-bit floating-point number formatted according to
// ECMA-262, 10th edition, section 7.1.12.1 and RFC 8785, section 3.2.2.3.
// with the exception that -0 is still formatted as -0.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
switch {
case math.Float64bits(n) == 0:
return zeroNumber
case math.IsNaN(n):
return nanString
case math.IsInf(n, +1):
return pinfString
case math.IsInf(n, -1):
return ninfString
}
return Token{str: "f", num: math.Float64bits(n)}
}
// Int constructs a Token representing a JSON number from an int64.
func Int(n int64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "i", num: uint64(n)}
}
// Uint constructs a Token representing a JSON number from a uint64.
func Uint(n uint64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "u", num: uint64(n)}
}
// Clone returns a copy of the token with a value that is not backed by the
// [Decoder] buffer and therefore remains valid past subsequent [Decoder] calls.
// It has no effect on tokens produced by constructor functions,
// since those are already self-contained.
func (t Token) Clone() Token {
// TODO: Allow caller to avoid any allocations?
if raw := t.raw; raw != nil {
// Avoid copying globals.
if t.raw.prevStart == 0 {
switch t.raw {
case Null.raw:
return Null
case False.raw:
return False
case True.raw:
return True
case BeginObject.raw:
return BeginObject
case EndObject.raw:
return EndObject
case BeginArray.raw:
return BeginArray
case EndArray.raw:
return EndArray
}
}
// This only ever needs to clone strings and numbers.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := bytes.Clone(raw.previousBuffer())
return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
}
return t
}
// Bool returns the value for a JSON boolean.
// It panics if the token kind is not a JSON boolean.
func (t Token) Bool() bool {
switch t.raw {
case True.raw:
return true
case False.raw:
return false
default:
panic("invalid JSON token kind: " + t.Kind().String())
}
}
// appendString appends a JSON string to dst and returns it.
// It panics if t is not a JSON string.
func (t Token) appendString(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw string value.
buf := raw.previousBuffer()
if Kind(buf[0]) == '"' {
if jsonwire.ConsumeSimpleString(buf) == len(buf) {
return append(dst, buf...), nil
}
dst, _, err := jsonwire.ReformatString(dst, buf, flags)
return dst, err
}
} else if len(t.str) != 0 && t.num == 0 {
// Handle exact string value.
return jsonwire.AppendQuote(dst, []byte(t.str), flags)
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// String returns the unescaped string value for a JSON string.
// For other JSON kinds, this returns the raw JSON representation.
func (t Token) String() string {
// This is inlinable to take advantage of "function outlining".
// This avoids an allocation for the string(b) conversion
// if the caller does not use the string in an escaping manner.
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
s, b := t.string()
if len(b) > 0 {
return string(b)
}
return s
}
func (t Token) string() (string, []byte) {
if raw := t.raw; raw != nil {
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if buf[0] == '"' {
// TODO: Preserve ValueFlags in Token?
isVerbatim := jsonwire.ConsumeSimpleString(buf) == len(buf)
return "", jsonwire.UnquoteMayCopy(buf, isVerbatim)
}
// Handle tokens that are not JSON strings for fmt.Stringer.
return "", buf
}
if len(t.str) != 0 && t.num == 0 {
return t.str, nil
}
// Handle tokens that are not JSON strings for fmt.Stringer.
if t.num > 0 {
switch t.str[0] {
case 'F':
return string(jsonwire.AppendFloat(nil, float64(math.Float32frombits(uint32(t.num))), 32)), nil
case 'f':
return string(jsonwire.AppendFloat(nil, float64(math.Float64frombits(uint64(t.num))), 64)), nil
case 'i':
return strconv.FormatInt(int64(t.num), 10), nil
case 'u':
return strconv.FormatUint(uint64(t.num), 10), nil
}
}
return "<invalid jsontext.Token>", nil
}
// appendNumber appends a JSON number to dst and returns it.
// It panics if t is not a JSON number.
func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw number value.
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
dst, _, err := jsonwire.ReformatNumber(dst, buf, flags)
return dst, err
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'F':
return jsonwire.AppendFloat(dst, float64(math.Float32frombits(uint32(t.num))), 32), nil
case 'f':
return jsonwire.AppendFloat(dst, float64(math.Float64frombits(uint64(t.num))), 64), nil
case 'i':
return strconv.AppendInt(dst, int64(t.num), 10), nil
case 'u':
return strconv.AppendUint(dst, uint64(t.num), 10), nil
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Float32 returns the floating-point value for a JSON number
// parsed according to 32 bits of precision.
//
// If the JSON number is outside the representable range of a float32,
// it returns +Inf or -Inf along with an error
// that matches [strconv.ErrRange] according to [errors.Is].
//
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
//
// It panics if the token kind is not a JSON number
// or a JSON string with the aforementioned values.
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers.
// This method should only be used if the caller knows
// from other context that this token is a JSON number
// formatted only to 32 bits of precision (such as being encoded
// using the [Float32] constructor). For all other situations,
// prefer using the [Token.Float] accessor instead.
func (t Token) Float32() (float32, error) {
f, err := t.float(32)
return float32(f), err
}
// Float returns the floating-point value for a JSON number
// parsed according to 64 bits of precision.
//
// If the JSON number is outside the representable range of a float64,
// it returns +Inf or -Inf along with an error
// that matches [strconv.ErrRange] according to [errors.Is].
//
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
//
// It panics if the token kind is not a JSON number
// or a JSON string with the aforementioned values.
func (t Token) Float() (float64, error) {
f, err := t.float(64)
return float64(f), err
}
func (t Token) float(bits int) (float64, error) {
if raw := t.raw; raw != nil {
// Handle raw JSON number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
fv, err := strconv.ParseFloat(string(buf), bits)
if err != nil {
err = &numError{accessor: "Float", value: t.String(), err: errors.Unwrap(err)} // only ever ErrRange
}
return fv, err
}
} else if t.num != 0 {
// Handle typed Go number value.
switch t.str[0] {
case 'F':
return float64(math.Float32frombits(uint32(t.num))), nil
case 'f':
f64 := float64(math.Float64frombits(uint64(t.num)))
if bits == 32 && !math.IsInf(f64, 0) && math.IsInf(float64(float32(f64)), 0) {
return f64, &numError{accessor: "Float", value: t.String(), err: strconv.ErrRange}
}
return f64, nil
case 'i':
return float64(int64(t.num)), nil // NOTE: This may lead to loss of precision.
case 'u':
return float64(uint64(t.num)), nil // NOTE: This may lead to loss of precision.
}
}
// Handle string values with "NaN", "Infinity", or "-Infinity".
if t.Kind() == '"' {
switch t.String() {
case "NaN":
return math.NaN(), nil
case "Infinity":
return math.Inf(+1), nil
case "-Infinity":
return math.Inf(-1), nil
}
// TODO: Should this be an error instead of a panic?
// We can safely switch from a panic to an error in the future.
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Int returns the signed integer value for a JSON number.
//
// It reports an error that matches [strconv.ErrSyntax] according to [errors.Is]
// if the JSON number does not match the restricted grammar of just a signed integer.
// It reports an error that matches [strconv.ErrRange] according to [errors.Is]
// if the JSON number is a signed integer, but outside the range of an int64.
// Even if an error is reported, a reasonable value is still returned.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an int64 will be saturated
// to the closest representable value.
//
// It panics if the token kind is not a JSON number.
func (t Token) Int() (int64, error) {
if raw := t.raw; raw != nil {
// Handle raw JSON number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if len(buf) > 0 && buf[0] == '-' {
// Prospectively parse a negative integer.
switch abs, ok := jsonwire.ParseUint(buf[len("-"):]); {
case abs > -minInt64:
return minInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
case ok:
return -1 * int64(abs), nil
}
} else {
// Prospectively parse a non-negative integer.
switch abs, ok := jsonwire.ParseUint(buf); {
case abs > +maxInt64:
return maxInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
case ok:
return +1 * int64(abs), nil
}
}
// This is not a signed integer, which implies ErrSyntax.
if Kind(buf[0]).normalize() == '0' {
f64, _ := strconv.ParseFloat(string(buf), 64)
return f64toi64(f64), &numError{accessor: "Int", value: t.String(), err: strconv.ErrSyntax}
}
} else if t.num != 0 {
// Handle typed Go number value.
switch t.str[0] {
case 'i':
return int64(t.num), nil
case 'u':
if t.num > maxInt64 {
return maxInt64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
}
return int64(t.num), nil
case 'f', 'F':
f64 := float64(math.Float64frombits(uint64(t.num)))
if t.str[0] == 'F' {
f64 = float64(math.Float32frombits(uint32(t.num)))
}
switch i64 := f64toi64(f64); {
case math.IsNaN(f64), math.Trunc(f64) != f64:
return i64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrSyntax}
case (i64 == minInt64 && f64 < minInt64) || (i64 == maxInt64 && f64 > maxInt64):
return i64, &numError{accessor: "Int", value: t.String(), err: strconv.ErrRange}
default:
return i64, nil
}
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
func f64toi64(f64 float64) int64 {
switch {
case math.IsNaN(f64):
return 0
case f64 >= maxInt64+1:
return maxInt64
case f64 < minInt64:
return minInt64
default:
return int64(f64) // NOTE: This may lead to loss of precision.
}
}
// Uint returns the unsigned integer value for a JSON number.
//
// It reports an error that matches [strconv.ErrSyntax] if the JSON number
// does not match the restricted grammar of just an unsigned integer.
// It reports an error that matches [strconv.ErrRange] if the JSON number
// is an unsigned integer, but outside the representable range of a uint64.
// Even if an error is reported, a reasonable value is still returned.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of a uint64 will be saturated
// to the closest representable value.
//
// It panics if the token kind is not a JSON number.
func (t Token) Uint() (uint64, error) {
// NOTE: This accessor returns 0 for any negative JSON number,
// which might be surprising, but is at least consistent with the behavior
// of saturating out-of-bounds numbers to the closest representable number.
// We report ErrSyntax instead of ErrRange since the grammar for
// an unsigned integer does not permit a negative sign.
if raw := t.raw; raw != nil {
// Handle raw JSON number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
// Prospectively parse an unsigned integer.
switch abs, ok := jsonwire.ParseUint(buf); {
case ok:
return abs, nil
case abs == maxUint64: // implies overflows
return maxUint64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrRange}
}
// This is not an unsigned integer, which implies ErrSyntax.
if Kind(buf[0]).normalize() == '0' {
f64, _ := strconv.ParseFloat(string(buf), 64)
return f64tou64(f64), &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
}
} else if t.num != 0 {
// Handle typed Go number value.
switch t.str[0] {
case 'u':
return t.num, nil
case 'i':
if int64(t.num) < minUint64 {
return minUint64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
}
return uint64(int64(t.num)), nil
case 'f', 'F':
f64 := float64(math.Float64frombits(uint64(t.num)))
if t.str[0] == 'F' {
f64 = float64(math.Float32frombits(uint32(t.num)))
}
switch u64 := f64tou64(f64); {
case math.IsNaN(f64), math.Trunc(f64) != f64, math.Signbit(f64):
return u64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrSyntax}
case (u64 == minUint64 && f64 < minUint64) || (u64 == maxUint64 && f64 > maxUint64):
return u64, &numError{accessor: "Uint", value: t.String(), err: strconv.ErrRange}
default:
return u64, nil
}
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
func f64tou64(f64 float64) uint64 {
switch {
case math.IsNaN(f64):
return 0
case f64 >= maxUint64+1:
return maxUint64
case f64 < minUint64:
return minUint64
default:
return uint64(f64) // NOTE: This may lead to loss of precision.
}
}
// Kind returns the token kind.
func (t Token) Kind() Kind {
switch {
case t.raw != nil:
raw := t.raw
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
return Kind(t.raw.buf[raw.prevStart]).normalize()
case t.num != 0:
return '0'
case len(t.str) != 0:
return '"'
default:
return invalidKind
}
}
// A Kind represents the kind of a JSON token.
//
// Kind represents each possible JSON token kind with a single byte,
// which is conveniently the first byte of that kind's grammar
// with the restriction that numbers always be represented with '0'.
type Kind byte
const (
KindInvalid Kind = 0 // invalid kind
KindNull Kind = 'n' // null
KindFalse Kind = 'f' // false
KindTrue Kind = 't' // true
KindString Kind = '"' // string
KindNumber Kind = '0' // number
KindBeginObject Kind = '{' // begin object
KindEndObject Kind = '}' // end object
KindBeginArray Kind = '[' // begin array
KindEndArray Kind = ']' // end array
)
const invalidKind Kind = 0
// String prints the kind in a humanly readable fashion.
func (k Kind) String() string {
switch k {
case 0:
return "invalid"
case 'n':
return "null"
case 'f':
return "false"
case 't':
return "true"
case '"':
return "string"
case '0':
return "number"
case '{':
return "{"
case '}':
return "}"
case '[':
return "["
case ']':
return "]"
default:
return "<invalid jsontext.Kind: " + jsonwire.QuoteRune([]byte{byte(k)}) + ">"
}
}
var normKind = [256]Kind{
'n': 'n',
'f': 'f',
't': 't',
'"': '"',
'{': '{',
'}': '}',
'[': '[',
']': ']',
'-': '0',
'0': '0',
'1': '0',
'2': '0',
'3': '0',
'4': '0',
'5': '0',
'6': '0',
'7': '0',
'8': '0',
'9': '0',
}
// normalize coalesces all possible starting characters of a number as just '0',
// and converts all invalid kinds to 0.
func (k Kind) normalize() Kind {
// A lookup table keeps the inlining cost as low as possible.
return normKind[k]
}

View File

@@ -0,0 +1,414 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package jsontext
import (
"bytes"
"errors"
"io"
"slices"
"sync"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// AppendFloat appends src to dst as a JSON number per RFC 8259, section 6.
//
// Except for -0, which is formatted as -0 instead of 0,
// the output is identical to ECMA-262, 10th edition, section 7.1.12.1
// and (for 64-bit precision) identical to RFC 8785, section 3.2.2.3.
// The values NaN, +Inf, and -Inf will be represented as a JSON string
// with the values "NaN", "Infinity", and "-Infinity".
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers. As such, prefer using 64 bits
// of precision unless the recipient can know from other context
// that the encoded number uses 32 bits of precision.
func AppendFloat(dst []byte, src float64, bits int) []byte {
if bits != 32 && bits != 64 {
panic("illegal AppendFloat bit size")
}
return jsonwire.AppendFloat(dst, src, bits)
}
// AppendFormat formats the JSON value in src and appends it to dst
// according to the specified options.
// See [Value.Format] for more details about the formatting behavior.
//
// The dst and src may overlap.
// If an error is reported, then the entirety of src is appended to dst.
func AppendFormat[Bytes ~[]byte | ~string](dst []byte, src Bytes, opts ...Options) ([]byte, error) {
e := getBufferedEncoder(opts...)
defer putBufferedEncoder(e)
e.s.Flags.Set(jsonflags.OmitTopLevelNewline | 1)
if err := e.s.WriteValue(Value(src)); err != nil {
return append(dst, src...), err
}
return append(dst, e.s.Buf...), nil
}
// NOTE: Value is analogous to v1 json.RawMessage.
// Value represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - an entire JSON object (e.g., {"fizz":"buzz"} )
// - an entire JSON array (e.g., [1,2,3] )
//
// Value can represent entire array or object values, while [Token] cannot.
// Value may contain leading and/or trailing whitespace.
type Value []byte
// Clone returns a copy of v.
func (v Value) Clone() Value {
return bytes.Clone(v)
}
// String returns the string formatting of v.
func (v Value) String() string {
if v == nil {
return "null"
}
return string(v)
}
// IsValid reports whether the raw JSON value is syntactically valid
// according to the specified options.
//
// By default (if no options are specified), it validates according to RFC 7493.
// It verifies whether the input is properly encoded as UTF-8,
// that escape sequences within strings decode to valid Unicode codepoints, and
// that all names in each object are unique.
// It does not verify whether numbers are representable within the limits
// of any common numeric type (e.g., float64, int64, or uint64).
//
// Relevant options include:
// - [AllowDuplicateNames]
// - [AllowInvalidUTF8]
//
// All other options are ignored.
func (v Value) IsValid(opts ...Options) bool {
// TODO: Document support for [WithByteLimit] and [WithDepthLimit].
d := getBufferedDecoder(v, opts...)
defer putBufferedDecoder(d)
_, errVal := d.ReadValue()
_, errEOF := d.ReadToken()
return errVal == nil && errEOF == io.EOF
}
// Format formats the raw JSON value in place.
//
// By default (if no options are specified), it validates according to RFC 7493
// and produces the minimal JSON representation, where
// all whitespace is elided and JSON strings use the shortest encoding.
//
// Relevant options include:
// - [AllowDuplicateNames]
// - [AllowInvalidUTF8]
// - [EscapeForHTML]
// - [EscapeForJS]
// - [PreserveRawStrings]
// - [CanonicalizeRawInts]
// - [CanonicalizeRawFloats]
// - [ReorderRawObjects]
// - [SpaceAfterColon]
// - [SpaceAfterComma]
// - [Multiline]
// - [WithIndent]
// - [WithIndentPrefix]
//
// All other options are ignored.
//
// It is guaranteed to succeed if the value is valid according to the same options.
// If the value is already formatted, then the buffer is not mutated.
func (v *Value) Format(opts ...Options) error {
// TODO: Document support for [WithByteLimit] and [WithDepthLimit].
return v.format(opts, nil)
}
// format accepts two []Options to avoid the allocation of appending them together.
// It is equivalent to v.Format(append(opts1, opts2...)...).
func (v *Value) format(opts1, opts2 []Options) error {
e := getBufferedEncoder(opts1...)
defer putBufferedEncoder(e)
e.s.Join(opts2...)
e.s.Flags.Set(jsonflags.OmitTopLevelNewline | 1)
if err := e.s.WriteValue(*v); err != nil {
return err
}
if !bytes.Equal(*v, e.s.Buf) {
*v = append((*v)[:0], e.s.Buf...)
}
return nil
}
// Compact removes all whitespace from the raw JSON value.
//
// It does not reformat JSON strings or numbers to use any other representation.
// To maximize the set of JSON values that can be formatted,
// this permits values with duplicate names and invalid UTF-8.
//
// Compact is equivalent to calling [Value.Format] with the following options:
// - [AllowDuplicateNames](true)
// - [AllowInvalidUTF8](true)
// - [PreserveRawStrings](true)
//
// Any options specified by the caller are applied after the initial set
// and may deliberately override prior options.
func (v *Value) Compact(opts ...Options) error {
return v.format([]Options{
AllowDuplicateNames(true),
AllowInvalidUTF8(true),
PreserveRawStrings(true),
}, opts)
}
// Indent reformats the whitespace in the raw JSON value so that each element
// in a JSON object or array begins on an indented line according to the nesting.
//
// It does not reformat JSON strings or numbers to use any other representation.
// To maximize the set of JSON values that can be formatted,
// this permits values with duplicate names and invalid UTF-8.
//
// Indent is equivalent to calling [Value.Format] with the following options:
// - [AllowDuplicateNames](true)
// - [AllowInvalidUTF8](true)
// - [PreserveRawStrings](true)
// - [Multiline](true)
//
// Any options specified by the caller are applied after the initial set
// and may deliberately override prior options.
func (v *Value) Indent(opts ...Options) error {
return v.format([]Options{
AllowDuplicateNames(true),
AllowInvalidUTF8(true),
PreserveRawStrings(true),
Multiline(true),
}, opts)
}
// Canonicalize canonicalizes the raw JSON value according to the
// JSON Canonicalization Scheme (JCS) as defined by RFC 8785
// where it produces a stable representation of a JSON value.
//
// JSON strings are formatted to use their minimal representation,
// JSON numbers are formatted as double precision numbers according
// to some stable serialization algorithm.
// JSON object members are sorted in ascending order by name.
// All whitespace is removed.
//
// The output stability is dependent on the stability of the application data
// (see RFC 8785, Appendix E). It cannot produce stable output from
// fundamentally unstable input. For example, if the JSON value
// contains ephemeral data (e.g., a frequently changing timestamp),
// then the value is still unstable regardless of whether this is called.
//
// Canonicalize is equivalent to calling [Value.Format] with the following options:
// - [CanonicalizeRawInts](true)
// - [CanonicalizeRawFloats](true)
// - [ReorderRawObjects](true)
//
// Any options specified by the caller are applied after the initial set
// and may deliberately override prior options.
//
// Note that JCS treats all JSON numbers as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example, integer values
// beyond ±2⁵³ will lose their precision. To preserve the original representation
// of JSON integers, additionally set [CanonicalizeRawInts] to false:
//
// v.Canonicalize(jsontext.CanonicalizeRawInts(false))
func (v *Value) Canonicalize(opts ...Options) error {
return v.format([]Options{
CanonicalizeRawInts(true),
CanonicalizeRawFloats(true),
ReorderRawObjects(true),
}, opts)
}
// MarshalJSON returns v as the JSON encoding of v.
// It returns the stored value as the raw JSON output without any validation.
// If v is nil, then this returns a JSON null.
func (v Value) MarshalJSON() ([]byte, error) {
// NOTE: This matches the behavior of v1 json.RawMessage.MarshalJSON.
if v == nil {
return []byte("null"), nil
}
return v, nil
}
// UnmarshalJSON sets v as the JSON encoding of b.
// It stores a copy of the provided raw JSON input without any validation.
func (v *Value) UnmarshalJSON(b []byte) error {
// NOTE: This matches the behavior of v1 json.RawMessage.UnmarshalJSON.
if v == nil {
return errors.New("jsontext.Value: UnmarshalJSON on nil pointer")
}
*v = append((*v)[:0], b...)
return nil
}
// Kind returns the starting token kind.
// For a valid value, this will never include [KindEndObject] or [KindEndArray].
func (v Value) Kind() Kind {
if v := v[jsonwire.ConsumeWhitespace(v):]; len(v) > 0 {
return Kind(v[0]).normalize()
}
return invalidKind
}
const commaAndWhitespace = ", \n\r\t"
type objectMember struct {
// name is the unquoted name.
name []byte // e.g., "name"
// buffer is the entirety of the raw JSON object member
// starting from right after the previous member (or opening '{')
// until right after the member value.
buffer []byte // e.g., `, \n\r\t"name": "value"`
}
func (x objectMember) Compare(y objectMember) int {
if c := jsonwire.CompareUTF16(x.name, y.name); c != 0 {
return c
}
// With [AllowDuplicateNames] or [AllowInvalidUTF8],
// names could be identical, so also sort using the member value.
return jsonwire.CompareUTF16(
bytes.TrimLeft(x.buffer, commaAndWhitespace),
bytes.TrimLeft(y.buffer, commaAndWhitespace))
}
var objectMemberPool = sync.Pool{New: func() any { return new([]objectMember) }}
func getObjectMembers() *[]objectMember {
ns := objectMemberPool.Get().(*[]objectMember)
*ns = (*ns)[:0]
return ns
}
func putObjectMembers(ns *[]objectMember) {
if cap(*ns) < 1<<10 {
clear(*ns) // avoid pinning name and buffer
objectMemberPool.Put(ns)
}
}
// mustReorderObjects reorders in-place all object members in a JSON value,
// which must be valid otherwise it panics.
func mustReorderObjects(b []byte) {
// Obtain a buffered encoder just to use its internal buffer as
// a scratch buffer for reordering object members.
e2 := getBufferedEncoder()
defer putBufferedEncoder(e2)
// Disable unnecessary checks to syntactically parse the JSON value.
d := getBufferedDecoder(b)
defer putBufferedDecoder(d)
d.s.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1)
mustReorderObjectsFromDecoder(d, &e2.s.Buf) // per RFC 8785, section 3.2.3
}
// mustReorderObjectsFromDecoder recursively reorders all object members in place
// according to the ordering specified in RFC 8785, section 3.2.3.
//
// Pre-conditions:
// - The value is valid (i.e., no decoder errors should ever occur).
// - Initial call is provided a Decoder reading from the start of v.
//
// Post-conditions:
// - Exactly one JSON value is read from the Decoder.
// - All fully-parsed JSON objects are reordered by directly moving
// the members in the value buffer.
//
// The runtime is approximately O(n·log(n)) + O(m·log(m)),
// where n is len(v) and m is the total number of object members.
func mustReorderObjectsFromDecoder(d *Decoder, scratch *[]byte) {
switch tok, err := d.ReadToken(); tok.Kind() {
case '{':
// Iterate and collect the name and offsets for every object member.
members := getObjectMembers()
defer putObjectMembers(members)
var prevMember objectMember
isSorted := true
beforeBody := d.InputOffset() // offset after '{'
for d.PeekKind() != '}' {
beforeName := d.InputOffset()
var flags jsonwire.ValueFlags
name, _ := d.s.ReadValue(&flags)
name = jsonwire.UnquoteMayCopy(name, flags.IsVerbatim())
mustReorderObjectsFromDecoder(d, scratch)
afterValue := d.InputOffset()
currMember := objectMember{name, d.s.buf[beforeName:afterValue]}
if isSorted && len(*members) > 0 {
isSorted = objectMember.Compare(prevMember, currMember) < 0
}
*members = append(*members, currMember)
prevMember = currMember
}
afterBody := d.InputOffset() // offset before '}'
d.ReadToken()
// Sort the members; return early if it's already sorted.
if isSorted {
return
}
firstBufferBeforeSorting := (*members)[0].buffer
slices.SortFunc(*members, objectMember.Compare)
firstBufferAfterSorting := (*members)[0].buffer
// Append the reordered members to a new buffer,
// then copy the reordered members back over the original members.
// Avoid swapping in place since each member may be a different size
// where moving a member over a smaller member may corrupt the data
// for subsequent members before they have been moved.
//
// The following invariant must hold:
// sum([m.after-m.before for m in members]) == afterBody-beforeBody
commaAndWhitespacePrefix := func(b []byte) []byte {
return b[:len(b)-len(bytes.TrimLeft(b, commaAndWhitespace))]
}
sorted := (*scratch)[:0]
for i, member := range *members {
switch {
case i == 0 && &member.buffer[0] != &firstBufferBeforeSorting[0]:
// First member after sorting is not the first member before sorting,
// so use the prefix of the first member before sorting.
sorted = append(sorted, commaAndWhitespacePrefix(firstBufferBeforeSorting)...)
sorted = append(sorted, bytes.TrimLeft(member.buffer, commaAndWhitespace)...)
case i != 0 && &member.buffer[0] == &firstBufferBeforeSorting[0]:
// Later member after sorting is the first member before sorting,
// so use the prefix of the first member after sorting.
sorted = append(sorted, commaAndWhitespacePrefix(firstBufferAfterSorting)...)
sorted = append(sorted, bytes.TrimLeft(member.buffer, commaAndWhitespace)...)
default:
sorted = append(sorted, member.buffer...)
}
}
if int(afterBody-beforeBody) != len(sorted) {
panic("BUG: length invariant violated")
}
copy(d.s.buf[beforeBody:afterBody], sorted)
// Update scratch buffer to the largest amount ever used.
if len(sorted) > len(*scratch) {
*scratch = sorted
}
case '[':
for d.PeekKind() != ']' {
mustReorderObjectsFromDecoder(d, scratch)
}
d.ReadToken()
default:
if err != nil {
panic("BUG: " + err.Error())
}
}
}

View File

@@ -0,0 +1,60 @@
#!/usr/bin/env bash
GOROOT=${1:-../go}
JSONROOT="."
cp $JSONROOT/alias_gen.go $JSONROOT/alias_gen.go.bak
cp $JSONROOT/options_format_global.go $JSONROOT/options_format_global.go.bak
cp $JSONROOT/options_format_test.go $JSONROOT/options_format_test.go.bak
rm -r $JSONROOT/*.go $JSONROOT/internal $JSONROOT/jsontext $JSONROOT/v1
mv $JSONROOT/alias_gen.go.bak $JSONROOT/alias_gen.go
cp -r $GOROOT/src/encoding/json/v2/*.go $JSONROOT/
cp -r $GOROOT/src/encoding/json/internal/ $JSONROOT/internal/
cp -r $GOROOT/src/encoding/json/jsontext/ $JSONROOT/jsontext/
mkdir $JSONROOT/v1
for X in $GOROOT/src/encoding/json/v2_*.go; do
cp $X $JSONROOT/v1/$(basename $X | sed "s/v2_//")
done
cd $JSONROOT
for X in $(git ls-files --cached --others --exclude-standard | grep ".*[.]go$"); do
if [ ! -e "$X" ]; then
continue
fi
sed -i 's/go:build goexperiment.jsonv2 && !goexperiment.jsonformat$/go:build (!goexperiment.jsonv2 || !go1.25) \&\& !goexperiment.jsonformat/' $X
sed -i 's/go:build goexperiment.jsonv2 && goexperiment.jsonformat$/go:build (!goexperiment.jsonv2 || !go1.25) \&\& goexperiment.jsonformat/' $X
sed -i 's/go:build goexperiment.jsonv2$/go:build !goexperiment.jsonv2 || !go1.25/' $X
sed -i 's|"encoding/json/v2"|"github.com/go-json-experiment/json"|' $X
sed -i 's|"encoding/json/internal"|"github.com/go-json-experiment/json/internal"|' $X
sed -i 's|"encoding/json/internal/jsonflags"|"github.com/go-json-experiment/json/internal/jsonflags"|' $X
sed -i 's|"encoding/json/internal/jsonopts"|"github.com/go-json-experiment/json/internal/jsonopts"|' $X
sed -i 's|"encoding/json/internal/jsontest"|"github.com/go-json-experiment/json/internal/jsontest"|' $X
sed -i 's|"encoding/json/internal/jsonwire"|"github.com/go-json-experiment/json/internal/jsonwire"|' $X
sed -i 's|"encoding/json/jsontext"|"github.com/go-json-experiment/json/jsontext"|' $X
sed -i 's|"encoding/json"|"github.com/go-json-experiment/json/v1"|' $X
sed -i 's|"internal/zstd"|"github.com/go-json-experiment/json/internal/zstd"|' $X
goimports -w $X
done
sed -i 's/v2[.]struct/json.struct/' $JSONROOT/errors_test.go
sed -i 's|jsonv1 "github.com/go-json-experiment/json/v1"|jsonv1 "encoding/json"|' $JSONROOT/bench_test.go
sed -i '/testenv/d' $JSONROOT/jsontext/token_test.go
# Remove documentation that only makes sense within the stdlib.
sed -i '/This package .* is experimental/,+4d' $JSONROOT/doc.go
sed -i '/This package .* is experimental/,+4d' $JSONROOT/jsontext/doc.go
git checkout internal/zstd # we still need local copy of zstd for testing
go run alias_gen.go "encoding/json" $JSONROOT/v1
go run alias_gen.go "encoding/json/v2" $JSONROOT
go run alias_gen.go "encoding/json/jsontext" $JSONROOT/jsontext
mv $JSONROOT/options_format_global.go.bak $JSONROOT/options_format_global.go
mv $JSONROOT/options_format_test.go.bak $JSONROOT/options_format_test.go
cp $GOROOT/src/encoding/json/internal/jsonopts/options_format.go $JSONROOT/options_format.go
sed -i '/go:build goexperiment.jsonv2/d' $JSONROOT/options_format.go
sed -i 's/package jsonopts/package json/' $JSONROOT/options_format.go
goimports -w $JSONROOT/options_format.go
sed -i -E '/^func (Marshal|MarshalWrite|MarshalEncode|Unmarshal|UnmarshalRead|UnmarshalDecode)\(/a opts = mayAppendSupportFormatTag(opts)' $JSONROOT/alias.go
sed -i -E '/^func (Marshal|MarshalWrite|MarshalEncode|Unmarshal|UnmarshalRead|UnmarshalDecode)\(/a opts = mayAppendSupportFormatTag(opts)' $JSONROOT/arshal.go
goimports -w $JSONROOT/alias.go
goimports -w $JSONROOT/arshal.go

View File

@@ -0,0 +1,290 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !goexperiment.jsonv2 || !go1.25
package json
import (
"fmt"
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
)
// Options configure [Marshal], [MarshalWrite], [MarshalEncode],
// [Unmarshal], [UnmarshalRead], and [UnmarshalDecode] with specific features.
// Each function takes in a variadic list of options, where properties
// set in later options override the value of previously set properties.
//
// The Options type is identical to [encoding/json.Options] and
// [encoding/json/jsontext.Options]. Options from the other packages can
// be used interchangeably with functionality in this package.
//
// Options represent either a singular option or a set of options.
// It can be functionally thought of as a Go map of option properties
// (even though the underlying implementation avoids Go maps for performance).
//
// The constructors (e.g., [Deterministic]) return a singular option value:
//
// opt := Deterministic(true)
//
// which is analogous to creating a single entry map:
//
// opt := Options{"Deterministic": true}
//
// [JoinOptions] composes multiple options values together:
//
// out := JoinOptions(opts...)
//
// which is analogous to making a new map and copying the options over:
//
// out := make(Options)
// for _, m := range opts {
// for k, v := range m {
// out[k] = v
// }
// }
//
// [GetOption] looks up the value of an options parameter:
//
// v, ok := GetOption(opts, Deterministic)
//
// which is analogous to a Go map lookup:
//
// v, ok := Options["Deterministic"]
//
// There is a single Options type, which is used with both marshal and unmarshal.
// Some options affect both operations, while others only affect one operation:
//
// - [StringifyNumbers] affects marshaling and unmarshaling
// - [Deterministic] affects marshaling only
// - [FormatNilSliceAsNull] affects marshaling only
// - [FormatNilMapAsNull] affects marshaling only
// - [OmitZeroStructFields] affects marshaling only
// - [MatchCaseInsensitiveNames] affects marshaling and unmarshaling
// - [RejectUnknownMembers] affects unmarshaling only
// - [WithMarshalers] affects marshaling only
// - [WithUnmarshalers] affects unmarshaling only
//
// Options that do not affect a particular operation are ignored.
type Options = jsonopts.Options
// JoinOptions coalesces the provided list of options into a single Options.
// Properties set in later options override the value of previously set properties.
func JoinOptions(srcs ...Options) Options {
var dst jsonopts.Struct
dst.Join(srcs...)
return &dst
}
// GetOption returns the value stored in opts with the provided setter,
// reporting whether the value is present.
// If not present, the returned value is the zero value for type T.
//
// Example usage:
//
// v, ok := json.GetOption(opts, json.Deterministic)
//
// Options are most commonly introspected to alter the JSON representation of
// [MarshalerTo.MarshalJSONTo] and [UnmarshalerFrom.UnmarshalJSONFrom] methods, and
// [MarshalToFunc] and [UnmarshalFromFunc] functions.
// In such cases, the presence bit should generally be ignored.
func GetOption[T any](opts Options, setter func(T) Options) (T, bool) {
return jsonopts.GetOption(opts, setter)
}
// DefaultOptionsV2 is the full set of all options that define v2 semantics.
// It is equivalent to the set of options in [encoding/json.DefaultOptionsV1]
// all being set to false. All other options are not present.
func DefaultOptionsV2() Options {
return &jsonopts.DefaultOptionsV2
}
// StringifyNumbers specifies that types that would normally be
// encoded as a JSON number to instead be encoded as a JSON string
// containing the equivalent JSON number value.
// When unmarshaling, the value is parsed from a JSON string
// containing the JSON number without any surrounding whitespace.
//
// When the `string` tag option is specified on a Go struct field,
// this option is applied for the top-level JSON value for that field.
// Unless StringifyNumbers was applied globally, the option does not
// recursively apply to nested JSON numbers within a JSON object or array.
// A Go type with custom marshal/unmarshal that represents a JSON number
// should respect the StringifyNumbers option and if specified
// serialize as a JSON number within a JSON string.
//
// According to RFC 8259, section 6, a JSON implementation may choose to
// limit the representation of a JSON number to an IEEE 754 binary64 value.
// This may cause decoders to lose precision for int64 and uint64 types.
// Quoting JSON numbers as a JSON string preserves the exact precision.
//
// This affects either marshaling or unmarshaling.
func StringifyNumbers(v bool) Options {
if v {
return jsonflags.StringifyNumbers | 1
} else {
return jsonflags.StringifyNumbers | 0
}
}
// Deterministic specifies that the same input value will be serialized
// as the exact same output bytes. Different processes of
// the same program will serialize equal values to the same bytes,
// but different versions of the same program are not guaranteed
// to produce the exact same sequence of bytes.
//
// This only affects marshaling and is ignored when unmarshaling.
func Deterministic(v bool) Options {
if v {
return jsonflags.Deterministic | 1
} else {
return jsonflags.Deterministic | 0
}
}
// FormatNilSliceAsNull specifies that a nil Go slice should marshal as a
// JSON null instead of the default representation as an empty JSON array
// (or an empty JSON string in the case of ~[]byte).
//
// This only affects marshaling and is ignored when unmarshaling.
func FormatNilSliceAsNull(v bool) Options {
if v {
return jsonflags.FormatNilSliceAsNull | 1
} else {
return jsonflags.FormatNilSliceAsNull | 0
}
}
// FormatNilMapAsNull specifies that a nil Go map should marshal as a
// JSON null instead of the default representation as an empty JSON object.
//
// This only affects marshaling and is ignored when unmarshaling.
func FormatNilMapAsNull(v bool) Options {
if v {
return jsonflags.FormatNilMapAsNull | 1
} else {
return jsonflags.FormatNilMapAsNull | 0
}
}
// OmitZeroStructFields specifies that a Go struct should marshal in such a way
// that all struct fields that are zero are omitted from the marshaled output
// if the value is zero as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This is semantically equivalent to specifying the `omitzero` tag option
// on every field in a Go struct.
//
// This only affects marshaling and is ignored when unmarshaling.
func OmitZeroStructFields(v bool) Options {
if v {
return jsonflags.OmitZeroStructFields | 1
} else {
return jsonflags.OmitZeroStructFields | 0
}
}
// MatchCaseInsensitiveNames specifies that JSON object members are matched
// against Go struct fields using a case-insensitive match of the name.
// If a name matches multiple fields, it chooses the field with an exact
// match of the name, otherwise it reports an error.
// Go struct fields explicitly marked with `case:strict` or `case:ignore`
// always use case-sensitive (or case-insensitive) name matching,
// regardless of the value of this option.
//
// This affects either marshaling or unmarshaling.
//
// By matching names in a case-insensitive manner, it also affects the detection
// of duplicate names (assuming [jsontext.AllowDuplicateNames] is false) since
// variations of the same name may match the same Go struct field.
// For example, when unmarshaling, the names "foo" and "Foo" may both
// match the same Go struct field and therefore be considered a duplicate name.
// When marshaling, normally it is impossible for any two Go struct fields to
// serialize in a way where they unmarshal into the same Go struct field
// since they all have unique exact names.
// However, with the use of an embedded fallback, it is possible for the
// embedded fallback to contain a name that also matches the name for
// a Go struct field, resulting in a duplicate name error.
func MatchCaseInsensitiveNames(v bool) Options {
if v {
return jsonflags.MatchCaseInsensitiveNames | 1
} else {
return jsonflags.MatchCaseInsensitiveNames | 0
}
}
// RejectUnknownMembers specifies that unknown members should be rejected
// when unmarshaling a JSON object.
//
// This only affects unmarshaling and is ignored when marshaling.
func RejectUnknownMembers(v bool) Options {
if v {
return jsonflags.RejectUnknownMembers | 1
} else {
return jsonflags.RejectUnknownMembers | 0
}
}
// WithMarshalers specifies a list of type-specific marshalers to use,
// which can be used to override the default marshal behavior for values
// of particular types.
//
// This only affects marshaling and is ignored when unmarshaling.
func WithMarshalers(v *Marshalers) Options {
return (*marshalersOption)(v)
}
// WithUnmarshalers specifies a list of type-specific unmarshalers to use,
// which can be used to override the default unmarshal behavior for values
// of particular types.
//
// This only affects unmarshaling and is ignored when marshaling.
func WithUnmarshalers(v *Unmarshalers) Options {
return (*unmarshalersOption)(v)
}
// These option types are declared here instead of "jsonopts"
// to avoid a dependency on "reflect" from "jsonopts".
type (
marshalersOption Marshalers
unmarshalersOption Unmarshalers
)
func (*marshalersOption) JSONOptions(internal.NotForPublicUse) {}
func (*unmarshalersOption) JSONOptions(internal.NotForPublicUse) {}
// Inject support into "jsonopts" to handle these types.
func init() {
jsonopts.GetUnknownOption = func(src jsonopts.Struct, zero jsonopts.Options) (any, bool) {
switch zero.(type) {
case *marshalersOption:
if !src.Flags.Has(jsonflags.Marshalers) {
return (*Marshalers)(nil), false
}
return src.Marshalers.(*Marshalers), true
case *unmarshalersOption:
if !src.Flags.Has(jsonflags.Unmarshalers) {
return (*Unmarshalers)(nil), false
}
return src.Unmarshalers.(*Unmarshalers), true
default:
panic(fmt.Sprintf("unknown option %T", zero))
}
}
jsonopts.JoinUnknownOption = func(dst jsonopts.Struct, src jsonopts.Options) jsonopts.Struct {
switch src := src.(type) {
case *marshalersOption:
dst.Flags.Set(jsonflags.Marshalers | 1)
dst.Marshalers = (*Marshalers)(src)
case *unmarshalersOption:
dst.Flags.Set(jsonflags.Unmarshalers | 1)
dst.Unmarshalers = (*Unmarshalers)(src)
default:
panic(fmt.Sprintf("unknown option %T", src))
}
return dst
}
}

View File

@@ -0,0 +1,122 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
// NOTE: While [ExperimentalSupportFormatTag] is exported,
// it is in an internal package and thus inaccessible for public use.
//
// The [github.com/go-json-experiment/json] module is kept in sync
// with the Go standard standard library and will expose this option
// in a way that public code can now directly reference.
type supportFormatTag struct {
Options
v bool
}
var (
enableFormatTag = supportFormatTag{v: true}
disableFormatTag = supportFormatTag{v: false}
)
// ExperimentalSupportFormatTag is a marker method that is specially
// recognized by the "encoding/json/v2" runtime.
func (o *supportFormatTag) ExperimentalSupportFormatTag() bool {
return o.v
}
// ExperimentalSupportFormatTag enables support for the `format` tag.
//
// WARNING: This is an experimental feature and will be removed in the future
// as either a failed experiment or be formally included in "encoding/json/v2"
// in some semantically similar form, in which case, users of this option
// must migrate to the officially supported feature.
//
// The `format` tag was originally part of the "encoding/json/v2" experiment
// but support for it was removed (see https://go.dev/issue/79071) for the
// initial release of "encoding/json/v2" in light of the anticipation that
// the Go language might support typed struct tags (https://go.dev/issue/74472).
//
// Typed struct tags are a more expressive and type-safe way to express format
// attributes than the bespoke `format` tag option that implements a miniature
// domain-specific language (DSL) within the "json" package itself.
//
// While "encoding/json/v2" was in the experimental phase,
// some users were already depending on the `format` tag option.
// This experimental option exists to provide a temporary workaround
// until the (hopeful) inclusion of typed struct tags and support for
// formatting directives in "encoding/json/v2" using that language mechanism.
//
// This option enables support for the `format` tag option, which specifies
// a format flag used to specialize the formatting of the field value.
// The option is a key-value pair specified as "format:value" where
// the value must be either a literal consisting of letters and numbers
// (e.g., `format:RFC3339`) or a single-quoted string literal
// (e.g., `format:'2006-01-02'`). The interpretation of the format flag
// is determined by the struct field type.
//
// Go types with alternative representations are as follows:
//
// - A Go []byte or [N]byte is usually represented as a JSON string
// containing the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the bytes value is represented as a JSON array
// where each element recursively uses the JSON representation of each byte.
//
// - A Go float is usually represented as a JSON number.
// If the format is "nonfinite", then NaN, +Inf, and -Inf are represented as
// the JSON strings "NaN", "Infinity", and "-Infinity", respectively.
// Without the use of this format, such string values result in a [SemanticError].
//
// - A nil Go map is usually encoded using an empty JSON object.
// If the format is "emitnull", then a nil map is encoded as a JSON null.
// If the format is "emitempty", then a nil map is encoded as an empty JSON object,
// regardless of whether [FormatNilMapAsNull] is specified.
//
// - A nil Go slice is usually encoded using an empty JSON array.
// If the format is "emitnull", then a nil slice is encoded as a JSON null.
// If the format is "emitempty", then a nil slice is encoded as an empty JSON array,
// regardless of whether [FormatNilSliceAsNull] is specified.
//
// - A Go pointer usually uses the JSON representation of the underlying value.
// The format is forwarded to the marshaling and unmarshaling of the underlying type.
//
// - A Go [time.Time] is usually represented as a JSON string containing
// the timestamp formatted in RFC 3339 with nanosecond precision.
// If the format matches one of the format constants declared
// in the time package (e.g., RFC1123), then that format is used.
// If the format is "unix", "unixmilli", "unixmicro", or "unixnano",
// then the timestamp is represented as a possibly fractional JSON number
// of the number of seconds (or milliseconds, microseconds, or nanoseconds)
// since the Unix epoch, which is January 1st, 1970 at 00:00:00 UTC.
// To avoid a fractional component when encoding,
// round the timestamp to the relevant unit.
// Otherwise if non-empty, the format is used as-is and
// encoded using [time.Time.Format] and
// decoded using [time.Time.Parse].
//
// - A Go [time.Duration] usually has no default representation.
// If the format is "sec", "milli", "micro", or "nano",
// then the duration is represented as a possibly fractional JSON number
// of the number of seconds (or milliseconds, microseconds, or nanoseconds).
// To avoid a fractional component when encoding,
// round the duration to the relevant unit.
// If the format is "units", it is represented as a JSON string
// encoded using [time.Duration.String] and decoded using [time.ParseDuration]
// (e.g., "1h30m" for 1 hour 30 minutes).
// If the format is "iso8601", it is represented as a JSON string using the
// ISO 8601 standard for durations (e.g., "PT1H30M" for 1 hour 30 minutes)
// using only accurate units of hours, minutes, and seconds.
func ExperimentalSupportFormatTag(v bool) Options {
if v {
return &enableFormatTag
} else {
return &disableFormatTag
}
}

View File

@@ -0,0 +1,30 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import "sync/atomic"
var globalEnableFormatTag atomic.Bool
// ExperimentalGlobalSupportFormatTag globally enables
// [ExperimentalSupportFormatTag] for all calls to
// [Marshal], [MarshalWrite], [MarshalEncode],
// [Unmarshal], [UnmarshalRead], or [UnmarshalDecode].
//
// WARNING: This is an experimental feature and will be removed in the future
// as either a failed experiment or be formally included in "github.com/go-json-experiment/json"
// in some semantically similar form, in which case, users of this option
// must migrate to the officially supported feature.
func ExperimentalGlobalSupportFormatTag(v bool) {
globalEnableFormatTag.Store(v)
}
func mayAppendSupportFormatTag(opts []Options) []Options {
if globalEnableFormatTag.Load() {
var optsArr [4]Options
opts = append(append(optsArr[:0], opts...), &enableFormatTag)
}
return opts
}

27
tools/tsgo/vendor/github.com/google/go-cmp/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,27 @@
Copyright (c) 2017 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@@ -0,0 +1,185 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cmpopts provides common options for the cmp package.
package cmpopts
import (
"errors"
"fmt"
"math"
"reflect"
"time"
"github.com/google/go-cmp/cmp"
)
func equateAlways(_, _ interface{}) bool { return true }
// EquateEmpty returns a [cmp.Comparer] option that determines all maps and slices
// with a length of zero to be equal, regardless of whether they are nil.
//
// EquateEmpty can be used in conjunction with [SortSlices] and [SortMaps].
func EquateEmpty() cmp.Option {
return cmp.FilterValues(isEmpty, cmp.Comparer(equateAlways))
}
func isEmpty(x, y interface{}) bool {
vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
return (x != nil && y != nil && vx.Type() == vy.Type()) &&
(vx.Kind() == reflect.Slice || vx.Kind() == reflect.Map) &&
(vx.Len() == 0 && vy.Len() == 0)
}
// EquateApprox returns a [cmp.Comparer] option that determines float32 or float64
// values to be equal if they are within a relative fraction or absolute margin.
// This option is not used when either x or y is NaN or infinite.
//
// The fraction determines that the difference of two values must be within the
// smaller fraction of the two values, while the margin determines that the two
// values must be within some absolute margin.
// To express only a fraction or only a margin, use 0 for the other parameter.
// The fraction and margin must be non-negative.
//
// The mathematical expression used is equivalent to:
//
// |x-y| ≤ max(fraction*min(|x|, |y|), margin)
//
// EquateApprox can be used in conjunction with [EquateNaNs].
func EquateApprox(fraction, margin float64) cmp.Option {
if margin < 0 || fraction < 0 || math.IsNaN(margin) || math.IsNaN(fraction) {
panic("margin or fraction must be a non-negative number")
}
a := approximator{fraction, margin}
return cmp.Options{
cmp.FilterValues(areRealF64s, cmp.Comparer(a.compareF64)),
cmp.FilterValues(areRealF32s, cmp.Comparer(a.compareF32)),
}
}
type approximator struct{ frac, marg float64 }
func areRealF64s(x, y float64) bool {
return !math.IsNaN(x) && !math.IsNaN(y) && !math.IsInf(x, 0) && !math.IsInf(y, 0)
}
func areRealF32s(x, y float32) bool {
return areRealF64s(float64(x), float64(y))
}
func (a approximator) compareF64(x, y float64) bool {
relMarg := a.frac * math.Min(math.Abs(x), math.Abs(y))
return math.Abs(x-y) <= math.Max(a.marg, relMarg)
}
func (a approximator) compareF32(x, y float32) bool {
return a.compareF64(float64(x), float64(y))
}
// EquateNaNs returns a [cmp.Comparer] option that determines float32 and float64
// NaN values to be equal.
//
// EquateNaNs can be used in conjunction with [EquateApprox].
func EquateNaNs() cmp.Option {
return cmp.Options{
cmp.FilterValues(areNaNsF64s, cmp.Comparer(equateAlways)),
cmp.FilterValues(areNaNsF32s, cmp.Comparer(equateAlways)),
}
}
func areNaNsF64s(x, y float64) bool {
return math.IsNaN(x) && math.IsNaN(y)
}
func areNaNsF32s(x, y float32) bool {
return areNaNsF64s(float64(x), float64(y))
}
// EquateApproxTime returns a [cmp.Comparer] option that determines two non-zero
// [time.Time] values to be equal if they are within some margin of one another.
// If both times have a monotonic clock reading, then the monotonic time
// difference will be used. The margin must be non-negative.
func EquateApproxTime(margin time.Duration) cmp.Option {
if margin < 0 {
panic("margin must be a non-negative number")
}
a := timeApproximator{margin}
return cmp.FilterValues(areNonZeroTimes, cmp.Comparer(a.compare))
}
func areNonZeroTimes(x, y time.Time) bool {
return !x.IsZero() && !y.IsZero()
}
type timeApproximator struct {
margin time.Duration
}
func (a timeApproximator) compare(x, y time.Time) bool {
// Avoid subtracting times to avoid overflow when the
// difference is larger than the largest representable duration.
if x.After(y) {
// Ensure x is always before y
x, y = y, x
}
// We're within the margin if x+margin >= y.
// Note: time.Time doesn't have AfterOrEqual method hence the negation.
return !x.Add(a.margin).Before(y)
}
// AnyError is an error that matches any non-nil error.
var AnyError anyError
type anyError struct{}
func (anyError) Error() string { return "any error" }
func (anyError) Is(err error) bool { return err != nil }
// EquateErrors returns a [cmp.Comparer] option that determines errors to be equal
// if [errors.Is] reports them to match. The [AnyError] error can be used to
// match any non-nil error.
func EquateErrors() cmp.Option {
return cmp.FilterValues(areConcreteErrors, cmp.Comparer(compareErrors))
}
// areConcreteErrors reports whether x and y are types that implement error.
// The input types are deliberately of the interface{} type rather than the
// error type so that we can handle situations where the current type is an
// interface{}, but the underlying concrete types both happen to implement
// the error interface.
func areConcreteErrors(x, y interface{}) bool {
_, ok1 := x.(error)
_, ok2 := y.(error)
return ok1 && ok2
}
func compareErrors(x, y interface{}) bool {
xe := x.(error)
ye := y.(error)
return errors.Is(xe, ye) || errors.Is(ye, xe)
}
// EquateComparable returns a [cmp.Option] that determines equality
// of comparable types by directly comparing them using the == operator in Go.
// The types to compare are specified by passing a value of that type.
// This option should only be used on types that are documented as being
// safe for direct == comparison. For example, [net/netip.Addr] is documented
// as being semantically safe to use with ==, while [time.Time] is documented
// to discourage the use of == on time values.
func EquateComparable(typs ...interface{}) cmp.Option {
types := make(typesFilter)
for _, typ := range typs {
switch t := reflect.TypeOf(typ); {
case !t.Comparable():
panic(fmt.Sprintf("%T is not a comparable Go type", typ))
case types[t]:
panic(fmt.Sprintf("%T is already specified", typ))
default:
types[t] = true
}
}
return cmp.FilterPath(types.filter, cmp.Comparer(equateAny))
}
type typesFilter map[reflect.Type]bool
func (tf typesFilter) filter(p cmp.Path) bool { return tf[p.Last().Type()] }
func equateAny(x, y interface{}) bool { return x == y }

View File

@@ -0,0 +1,206 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmpopts
import (
"fmt"
"reflect"
"unicode"
"unicode/utf8"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/internal/function"
)
// IgnoreFields returns an [cmp.Option] that ignores fields of the
// given names on a single struct type. It respects the names of exported fields
// that are forwarded due to struct embedding.
// The struct type is specified by passing in a value of that type.
//
// The name may be a dot-delimited string (e.g., "Foo.Bar") to ignore a
// specific sub-field that is embedded or nested within the parent struct.
func IgnoreFields(typ interface{}, names ...string) cmp.Option {
sf := newStructFilter(typ, names...)
return cmp.FilterPath(sf.filter, cmp.Ignore())
}
// IgnoreTypes returns an [cmp.Option] that ignores all values assignable to
// certain types, which are specified by passing in a value of each type.
func IgnoreTypes(typs ...interface{}) cmp.Option {
tf := newTypeFilter(typs...)
return cmp.FilterPath(tf.filter, cmp.Ignore())
}
type typeFilter []reflect.Type
func newTypeFilter(typs ...interface{}) (tf typeFilter) {
for _, typ := range typs {
t := reflect.TypeOf(typ)
if t == nil {
// This occurs if someone tries to pass in sync.Locker(nil)
panic("cannot determine type; consider using IgnoreInterfaces")
}
tf = append(tf, t)
}
return tf
}
func (tf typeFilter) filter(p cmp.Path) bool {
if len(p) < 1 {
return false
}
t := p.Last().Type()
for _, ti := range tf {
if t.AssignableTo(ti) {
return true
}
}
return false
}
// IgnoreInterfaces returns an [cmp.Option] that ignores all values or references of
// values assignable to certain interface types. These interfaces are specified
// by passing in an anonymous struct with the interface types embedded in it.
// For example, to ignore [sync.Locker], pass in struct{sync.Locker}{}.
func IgnoreInterfaces(ifaces interface{}) cmp.Option {
tf := newIfaceFilter(ifaces)
return cmp.FilterPath(tf.filter, cmp.Ignore())
}
type ifaceFilter []reflect.Type
func newIfaceFilter(ifaces interface{}) (tf ifaceFilter) {
t := reflect.TypeOf(ifaces)
if ifaces == nil || t.Name() != "" || t.Kind() != reflect.Struct {
panic("input must be an anonymous struct")
}
for i := 0; i < t.NumField(); i++ {
fi := t.Field(i)
switch {
case !fi.Anonymous:
panic("struct cannot have named fields")
case fi.Type.Kind() != reflect.Interface:
panic("embedded field must be an interface type")
case fi.Type.NumMethod() == 0:
// This matches everything; why would you ever want this?
panic("cannot ignore empty interface")
default:
tf = append(tf, fi.Type)
}
}
return tf
}
func (tf ifaceFilter) filter(p cmp.Path) bool {
if len(p) < 1 {
return false
}
t := p.Last().Type()
for _, ti := range tf {
if t.AssignableTo(ti) {
return true
}
if t.Kind() != reflect.Ptr && reflect.PtrTo(t).AssignableTo(ti) {
return true
}
}
return false
}
// IgnoreUnexported returns an [cmp.Option] that only ignores the immediate unexported
// fields of a struct, including anonymous fields of unexported types.
// In particular, unexported fields within the struct's exported fields
// of struct types, including anonymous fields, will not be ignored unless the
// type of the field itself is also passed to IgnoreUnexported.
//
// Avoid ignoring unexported fields of a type which you do not control (i.e. a
// type from another repository), as changes to the implementation of such types
// may change how the comparison behaves. Prefer a custom [cmp.Comparer] instead.
func IgnoreUnexported(typs ...interface{}) cmp.Option {
ux := newUnexportedFilter(typs...)
return cmp.FilterPath(ux.filter, cmp.Ignore())
}
type unexportedFilter struct{ m map[reflect.Type]bool }
func newUnexportedFilter(typs ...interface{}) unexportedFilter {
ux := unexportedFilter{m: make(map[reflect.Type]bool)}
for _, typ := range typs {
t := reflect.TypeOf(typ)
if t == nil || t.Kind() != reflect.Struct {
panic(fmt.Sprintf("%T must be a non-pointer struct", typ))
}
ux.m[t] = true
}
return ux
}
func (xf unexportedFilter) filter(p cmp.Path) bool {
sf, ok := p.Index(-1).(cmp.StructField)
if !ok {
return false
}
return xf.m[p.Index(-2).Type()] && !isExported(sf.Name())
}
// isExported reports whether the identifier is exported.
func isExported(id string) bool {
r, _ := utf8.DecodeRuneInString(id)
return unicode.IsUpper(r)
}
// IgnoreSliceElements returns an [cmp.Option] that ignores elements of []V.
// The discard function must be of the form "func(T) bool" which is used to
// ignore slice elements of type V, where V is assignable to T.
// Elements are ignored if the function reports true.
func IgnoreSliceElements(discardFunc interface{}) cmp.Option {
vf := reflect.ValueOf(discardFunc)
if !function.IsType(vf.Type(), function.ValuePredicate) || vf.IsNil() {
panic(fmt.Sprintf("invalid discard function: %T", discardFunc))
}
return cmp.FilterPath(func(p cmp.Path) bool {
si, ok := p.Index(-1).(cmp.SliceIndex)
if !ok {
return false
}
if !si.Type().AssignableTo(vf.Type().In(0)) {
return false
}
vx, vy := si.Values()
if vx.IsValid() && vf.Call([]reflect.Value{vx})[0].Bool() {
return true
}
if vy.IsValid() && vf.Call([]reflect.Value{vy})[0].Bool() {
return true
}
return false
}, cmp.Ignore())
}
// IgnoreMapEntries returns an [cmp.Option] that ignores entries of map[K]V.
// The discard function must be of the form "func(T, R) bool" which is used to
// ignore map entries of type K and V, where K and V are assignable to T and R.
// Entries are ignored if the function reports true.
func IgnoreMapEntries(discardFunc interface{}) cmp.Option {
vf := reflect.ValueOf(discardFunc)
if !function.IsType(vf.Type(), function.KeyValuePredicate) || vf.IsNil() {
panic(fmt.Sprintf("invalid discard function: %T", discardFunc))
}
return cmp.FilterPath(func(p cmp.Path) bool {
mi, ok := p.Index(-1).(cmp.MapIndex)
if !ok {
return false
}
if !mi.Key().Type().AssignableTo(vf.Type().In(0)) || !mi.Type().AssignableTo(vf.Type().In(1)) {
return false
}
k := mi.Key()
vx, vy := mi.Values()
if vx.IsValid() && vf.Call([]reflect.Value{k, vx})[0].Bool() {
return true
}
if vy.IsValid() && vf.Call([]reflect.Value{k, vy})[0].Bool() {
return true
}
return false
}, cmp.Ignore())
}

View File

@@ -0,0 +1,171 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmpopts
import (
"fmt"
"reflect"
"sort"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/internal/function"
)
// SortSlices returns a [cmp.Transformer] option that sorts all []V.
// The lessOrCompareFunc function must be either
// a less function of the form "func(T, T) bool" or
// a compare function of the format "func(T, T) int"
// which is used to sort any slice with element type V that is assignable to T.
//
// A less function must be:
// - Deterministic: less(x, y) == less(x, y)
// - Irreflexive: !less(x, x)
// - Transitive: if !less(x, y) and !less(y, z), then !less(x, z)
//
// A compare function must be:
// - Deterministic: compare(x, y) == compare(x, y)
// - Irreflexive: compare(x, x) == 0
// - Transitive: if !less(x, y) and !less(y, z), then !less(x, z)
//
// The function does not have to be "total". That is, if x != y, but
// less or compare report inequality, their relative order is maintained.
//
// SortSlices can be used in conjunction with [EquateEmpty].
func SortSlices(lessOrCompareFunc interface{}) cmp.Option {
vf := reflect.ValueOf(lessOrCompareFunc)
if (!function.IsType(vf.Type(), function.Less) && !function.IsType(vf.Type(), function.Compare)) || vf.IsNil() {
panic(fmt.Sprintf("invalid less or compare function: %T", lessOrCompareFunc))
}
ss := sliceSorter{vf.Type().In(0), vf}
return cmp.FilterValues(ss.filter, cmp.Transformer("cmpopts.SortSlices", ss.sort))
}
type sliceSorter struct {
in reflect.Type // T
fnc reflect.Value // func(T, T) bool
}
func (ss sliceSorter) filter(x, y interface{}) bool {
vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
if !(x != nil && y != nil && vx.Type() == vy.Type()) ||
!(vx.Kind() == reflect.Slice && vx.Type().Elem().AssignableTo(ss.in)) ||
(vx.Len() <= 1 && vy.Len() <= 1) {
return false
}
// Check whether the slices are already sorted to avoid an infinite
// recursion cycle applying the same transform to itself.
ok1 := sort.SliceIsSorted(x, func(i, j int) bool { return ss.less(vx, i, j) })
ok2 := sort.SliceIsSorted(y, func(i, j int) bool { return ss.less(vy, i, j) })
return !ok1 || !ok2
}
func (ss sliceSorter) sort(x interface{}) interface{} {
src := reflect.ValueOf(x)
dst := reflect.MakeSlice(src.Type(), src.Len(), src.Len())
for i := 0; i < src.Len(); i++ {
dst.Index(i).Set(src.Index(i))
}
sort.SliceStable(dst.Interface(), func(i, j int) bool { return ss.less(dst, i, j) })
ss.checkSort(dst)
return dst.Interface()
}
func (ss sliceSorter) checkSort(v reflect.Value) {
start := -1 // Start of a sequence of equal elements.
for i := 1; i < v.Len(); i++ {
if ss.less(v, i-1, i) {
// Check that first and last elements in v[start:i] are equal.
if start >= 0 && (ss.less(v, start, i-1) || ss.less(v, i-1, start)) {
panic(fmt.Sprintf("incomparable values detected: want equal elements: %v", v.Slice(start, i)))
}
start = -1
} else if start == -1 {
start = i
}
}
}
func (ss sliceSorter) less(v reflect.Value, i, j int) bool {
vx, vy := v.Index(i), v.Index(j)
vo := ss.fnc.Call([]reflect.Value{vx, vy})[0]
if vo.Kind() == reflect.Bool {
return vo.Bool()
} else {
return vo.Int() < 0
}
}
// SortMaps returns a [cmp.Transformer] option that flattens map[K]V types to be
// a sorted []struct{K, V}. The lessOrCompareFunc function must be either
// a less function of the form "func(T, T) bool" or
// a compare function of the format "func(T, T) int"
// which is used to sort any map with key K that is assignable to T.
//
// Flattening the map into a slice has the property that [cmp.Equal] is able to
// use [cmp.Comparer] options on K or the K.Equal method if it exists.
//
// A less function must be:
// - Deterministic: less(x, y) == less(x, y)
// - Irreflexive: !less(x, x)
// - Transitive: if !less(x, y) and !less(y, z), then !less(x, z)
// - Total: if x != y, then either less(x, y) or less(y, x)
//
// A compare function must be:
// - Deterministic: compare(x, y) == compare(x, y)
// - Irreflexive: compare(x, x) == 0
// - Transitive: if compare(x, y) < 0 and compare(y, z) < 0, then compare(x, z) < 0
// - Total: if x != y, then compare(x, y) != 0
//
// SortMaps can be used in conjunction with [EquateEmpty].
func SortMaps(lessOrCompareFunc interface{}) cmp.Option {
vf := reflect.ValueOf(lessOrCompareFunc)
if (!function.IsType(vf.Type(), function.Less) && !function.IsType(vf.Type(), function.Compare)) || vf.IsNil() {
panic(fmt.Sprintf("invalid less or compare function: %T", lessOrCompareFunc))
}
ms := mapSorter{vf.Type().In(0), vf}
return cmp.FilterValues(ms.filter, cmp.Transformer("cmpopts.SortMaps", ms.sort))
}
type mapSorter struct {
in reflect.Type // T
fnc reflect.Value // func(T, T) bool
}
func (ms mapSorter) filter(x, y interface{}) bool {
vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
return (x != nil && y != nil && vx.Type() == vy.Type()) &&
(vx.Kind() == reflect.Map && vx.Type().Key().AssignableTo(ms.in)) &&
(vx.Len() != 0 || vy.Len() != 0)
}
func (ms mapSorter) sort(x interface{}) interface{} {
src := reflect.ValueOf(x)
outType := reflect.StructOf([]reflect.StructField{
{Name: "K", Type: src.Type().Key()},
{Name: "V", Type: src.Type().Elem()},
})
dst := reflect.MakeSlice(reflect.SliceOf(outType), src.Len(), src.Len())
for i, k := range src.MapKeys() {
v := reflect.New(outType).Elem()
v.Field(0).Set(k)
v.Field(1).Set(src.MapIndex(k))
dst.Index(i).Set(v)
}
sort.Slice(dst.Interface(), func(i, j int) bool { return ms.less(dst, i, j) })
ms.checkSort(dst)
return dst.Interface()
}
func (ms mapSorter) checkSort(v reflect.Value) {
for i := 1; i < v.Len(); i++ {
if !ms.less(v, i-1, i) {
panic(fmt.Sprintf("partial order detected: want %v < %v", v.Index(i-1), v.Index(i)))
}
}
}
func (ms mapSorter) less(v reflect.Value, i, j int) bool {
vx, vy := v.Index(i).Field(0), v.Index(j).Field(0)
vo := ms.fnc.Call([]reflect.Value{vx, vy})[0]
if vo.Kind() == reflect.Bool {
return vo.Bool()
} else {
return vo.Int() < 0
}
}

View File

@@ -0,0 +1,189 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmpopts
import (
"fmt"
"reflect"
"strings"
"github.com/google/go-cmp/cmp"
)
// filterField returns a new Option where opt is only evaluated on paths that
// include a specific exported field on a single struct type.
// The struct type is specified by passing in a value of that type.
//
// The name may be a dot-delimited string (e.g., "Foo.Bar") to select a
// specific sub-field that is embedded or nested within the parent struct.
func filterField(typ interface{}, name string, opt cmp.Option) cmp.Option {
// TODO: This is currently unexported over concerns of how helper filters
// can be composed together easily.
// TODO: Add tests for FilterField.
sf := newStructFilter(typ, name)
return cmp.FilterPath(sf.filter, opt)
}
type structFilter struct {
t reflect.Type // The root struct type to match on
ft fieldTree // Tree of fields to match on
}
func newStructFilter(typ interface{}, names ...string) structFilter {
// TODO: Perhaps allow * as a special identifier to allow ignoring any
// number of path steps until the next field match?
// This could be useful when a concrete struct gets transformed into
// an anonymous struct where it is not possible to specify that by type,
// but the transformer happens to provide guarantees about the names of
// the transformed fields.
t := reflect.TypeOf(typ)
if t == nil || t.Kind() != reflect.Struct {
panic(fmt.Sprintf("%T must be a non-pointer struct", typ))
}
var ft fieldTree
for _, name := range names {
cname, err := canonicalName(t, name)
if err != nil {
panic(fmt.Sprintf("%s: %v", strings.Join(cname, "."), err))
}
ft.insert(cname)
}
return structFilter{t, ft}
}
func (sf structFilter) filter(p cmp.Path) bool {
for i, ps := range p {
if ps.Type().AssignableTo(sf.t) && sf.ft.matchPrefix(p[i+1:]) {
return true
}
}
return false
}
// fieldTree represents a set of dot-separated identifiers.
//
// For example, inserting the following selectors:
//
// Foo
// Foo.Bar.Baz
// Foo.Buzz
// Nuka.Cola.Quantum
//
// Results in a tree of the form:
//
// {sub: {
// "Foo": {ok: true, sub: {
// "Bar": {sub: {
// "Baz": {ok: true},
// }},
// "Buzz": {ok: true},
// }},
// "Nuka": {sub: {
// "Cola": {sub: {
// "Quantum": {ok: true},
// }},
// }},
// }}
type fieldTree struct {
ok bool // Whether this is a specified node
sub map[string]fieldTree // The sub-tree of fields under this node
}
// insert inserts a sequence of field accesses into the tree.
func (ft *fieldTree) insert(cname []string) {
if ft.sub == nil {
ft.sub = make(map[string]fieldTree)
}
if len(cname) == 0 {
ft.ok = true
return
}
sub := ft.sub[cname[0]]
sub.insert(cname[1:])
ft.sub[cname[0]] = sub
}
// matchPrefix reports whether any selector in the fieldTree matches
// the start of path p.
func (ft fieldTree) matchPrefix(p cmp.Path) bool {
for _, ps := range p {
switch ps := ps.(type) {
case cmp.StructField:
ft = ft.sub[ps.Name()]
if ft.ok {
return true
}
if len(ft.sub) == 0 {
return false
}
case cmp.Indirect:
default:
return false
}
}
return false
}
// canonicalName returns a list of identifiers where any struct field access
// through an embedded field is expanded to include the names of the embedded
// types themselves.
//
// For example, suppose field "Foo" is not directly in the parent struct,
// but actually from an embedded struct of type "Bar". Then, the canonical name
// of "Foo" is actually "Bar.Foo".
//
// Suppose field "Foo" is not directly in the parent struct, but actually
// a field in two different embedded structs of types "Bar" and "Baz".
// Then the selector "Foo" causes a panic since it is ambiguous which one it
// refers to. The user must specify either "Bar.Foo" or "Baz.Foo".
func canonicalName(t reflect.Type, sel string) ([]string, error) {
var name string
sel = strings.TrimPrefix(sel, ".")
if sel == "" {
return nil, fmt.Errorf("name must not be empty")
}
if i := strings.IndexByte(sel, '.'); i < 0 {
name, sel = sel, ""
} else {
name, sel = sel[:i], sel[i:]
}
// Type must be a struct or pointer to struct.
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return nil, fmt.Errorf("%v must be a struct", t)
}
// Find the canonical name for this current field name.
// If the field exists in an embedded struct, then it will be expanded.
sf, _ := t.FieldByName(name)
if !isExported(name) {
// Avoid using reflect.Type.FieldByName for unexported fields due to
// buggy behavior with regard to embeddeding and unexported fields.
// See https://golang.org/issue/4876 for details.
sf = reflect.StructField{}
for i := 0; i < t.NumField() && sf.Name == ""; i++ {
if t.Field(i).Name == name {
sf = t.Field(i)
}
}
}
if sf.Name == "" {
return []string{name}, fmt.Errorf("does not exist")
}
var ss []string
for i := range sf.Index {
ss = append(ss, t.FieldByIndex(sf.Index[:i+1]).Name)
}
if sel == "" {
return ss, nil
}
ssPost, err := canonicalName(sf.Type, sel)
return append(ss, ssPost...), err
}

View File

@@ -0,0 +1,36 @@
// Copyright 2018, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmpopts
import (
"github.com/google/go-cmp/cmp"
)
type xformFilter struct{ xform cmp.Option }
func (xf xformFilter) filter(p cmp.Path) bool {
for _, ps := range p {
if t, ok := ps.(cmp.Transform); ok && t.Option() == xf.xform {
return false
}
}
return true
}
// AcyclicTransformer returns a [cmp.Transformer] with a filter applied that ensures
// that the transformer cannot be recursively applied upon its own output.
//
// An example use case is a transformer that splits a string by lines:
//
// AcyclicTransformer("SplitLines", func(s string) []string{
// return strings.Split(s, "\n")
// })
//
// Had this been an unfiltered [cmp.Transformer] instead, this would result in an
// infinite cycle converting a string to []string to [][]string and so on.
func AcyclicTransformer(name string, xformFunc interface{}) cmp.Option {
xf := xformFilter{cmp.Transformer(name, xformFunc)}
return cmp.FilterPath(xf.filter, xf.xform)
}

View File

@@ -0,0 +1,671 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cmp determines equality of values.
//
// This package is intended to be a more powerful and safer alternative to
// [reflect.DeepEqual] for comparing whether two values are semantically equal.
// It is intended to only be used in tests, as performance is not a goal and
// it may panic if it cannot compare the values. Its propensity towards
// panicking means that its unsuitable for production environments where a
// spurious panic may be fatal.
//
// The primary features of cmp are:
//
// - When the default behavior of equality does not suit the test's needs,
// custom equality functions can override the equality operation.
// For example, an equality function may report floats as equal so long as
// they are within some tolerance of each other.
//
// - Types with an Equal method (e.g., [time.Time.Equal]) may use that method
// to determine equality. This allows package authors to determine
// the equality operation for the types that they define.
//
// - If no custom equality functions are used and no Equal method is defined,
// equality is determined by recursively comparing the primitive kinds on
// both values, much like [reflect.DeepEqual]. Unlike [reflect.DeepEqual],
// unexported fields are not compared by default; they result in panics
// unless suppressed by using an [Ignore] option
// (see [github.com/google/go-cmp/cmp/cmpopts.IgnoreUnexported])
// or explicitly compared using the [Exporter] option.
package cmp
import (
"fmt"
"reflect"
"strings"
"github.com/google/go-cmp/cmp/internal/diff"
"github.com/google/go-cmp/cmp/internal/function"
"github.com/google/go-cmp/cmp/internal/value"
)
// TODO(≥go1.18): Use any instead of interface{}.
// Equal reports whether x and y are equal by recursively applying the
// following rules in the given order to x and y and all of their sub-values:
//
// - Let S be the set of all [Ignore], [Transformer], and [Comparer] options that
// remain after applying all path filters, value filters, and type filters.
// If at least one [Ignore] exists in S, then the comparison is ignored.
// If the number of [Transformer] and [Comparer] options in S is non-zero,
// then Equal panics because it is ambiguous which option to use.
// If S contains a single [Transformer], then use that to transform
// the current values and recursively call Equal on the output values.
// If S contains a single [Comparer], then use that to compare the current values.
// Otherwise, evaluation proceeds to the next rule.
//
// - If the values have an Equal method of the form "(T) Equal(T) bool" or
// "(T) Equal(I) bool" where T is assignable to I, then use the result of
// x.Equal(y) even if x or y is nil. Otherwise, no such method exists and
// evaluation proceeds to the next rule.
//
// - Lastly, try to compare x and y based on their basic kinds.
// Simple kinds like booleans, integers, floats, complex numbers, strings,
// and channels are compared using the equivalent of the == operator in Go.
// Functions are only equal if they are both nil, otherwise they are unequal.
//
// Structs are equal if recursively calling Equal on all fields report equal.
// If a struct contains unexported fields, Equal panics unless an [Ignore] option
// (e.g., [github.com/google/go-cmp/cmp/cmpopts.IgnoreUnexported]) ignores that field
// or the [Exporter] option explicitly permits comparing the unexported field.
//
// Slices are equal if they are both nil or both non-nil, where recursively
// calling Equal on all non-ignored slice or array elements report equal.
// Empty non-nil slices and nil slices are not equal; to equate empty slices,
// consider using [github.com/google/go-cmp/cmp/cmpopts.EquateEmpty].
//
// Maps are equal if they are both nil or both non-nil, where recursively
// calling Equal on all non-ignored map entries report equal.
// Map keys are equal according to the == operator.
// To use custom comparisons for map keys, consider using
// [github.com/google/go-cmp/cmp/cmpopts.SortMaps].
// Empty non-nil maps and nil maps are not equal; to equate empty maps,
// consider using [github.com/google/go-cmp/cmp/cmpopts.EquateEmpty].
//
// Pointers and interfaces are equal if they are both nil or both non-nil,
// where they have the same underlying concrete type and recursively
// calling Equal on the underlying values reports equal.
//
// Before recursing into a pointer, slice element, or map, the current path
// is checked to detect whether the address has already been visited.
// If there is a cycle, then the pointed at values are considered equal
// only if both addresses were previously visited in the same path step.
func Equal(x, y interface{}, opts ...Option) bool {
s := newState(opts)
s.compareAny(rootStep(x, y))
return s.result.Equal()
}
// Diff returns a human-readable report of the differences between two values:
// y - x. It returns an empty string if and only if Equal returns true for the
// same input values and options.
//
// The output is displayed as a literal in pseudo-Go syntax.
// At the start of each line, a "-" prefix indicates an element removed from x,
// a "+" prefix to indicates an element added from y, and the lack of a prefix
// indicates an element common to both x and y. If possible, the output
// uses fmt.Stringer.String or error.Error methods to produce more humanly
// readable outputs. In such cases, the string is prefixed with either an
// 's' or 'e' character, respectively, to indicate that the method was called.
//
// Do not depend on this output being stable. If you need the ability to
// programmatically interpret the difference, consider using a custom Reporter.
func Diff(x, y interface{}, opts ...Option) string {
s := newState(opts)
// Optimization: If there are no other reporters, we can optimize for the
// common case where the result is equal (and thus no reported difference).
// This avoids the expensive construction of a difference tree.
if len(s.reporters) == 0 {
s.compareAny(rootStep(x, y))
if s.result.Equal() {
return ""
}
s.result = diff.Result{} // Reset results
}
r := new(defaultReporter)
s.reporters = append(s.reporters, reporter{r})
s.compareAny(rootStep(x, y))
d := r.String()
if (d == "") != s.result.Equal() {
panic("inconsistent difference and equality results")
}
return d
}
// rootStep constructs the first path step. If x and y have differing types,
// then they are stored within an empty interface type.
func rootStep(x, y interface{}) PathStep {
vx := reflect.ValueOf(x)
vy := reflect.ValueOf(y)
// If the inputs are different types, auto-wrap them in an empty interface
// so that they have the same parent type.
var t reflect.Type
if !vx.IsValid() || !vy.IsValid() || vx.Type() != vy.Type() {
t = anyType
if vx.IsValid() {
vvx := reflect.New(t).Elem()
vvx.Set(vx)
vx = vvx
}
if vy.IsValid() {
vvy := reflect.New(t).Elem()
vvy.Set(vy)
vy = vvy
}
} else {
t = vx.Type()
}
return &pathStep{t, vx, vy}
}
type state struct {
// These fields represent the "comparison state".
// Calling statelessCompare must not result in observable changes to these.
result diff.Result // The current result of comparison
curPath Path // The current path in the value tree
curPtrs pointerPath // The current set of visited pointers
reporters []reporter // Optional reporters
// recChecker checks for infinite cycles applying the same set of
// transformers upon the output of itself.
recChecker recChecker
// dynChecker triggers pseudo-random checks for option correctness.
// It is safe for statelessCompare to mutate this value.
dynChecker dynChecker
// These fields, once set by processOption, will not change.
exporters []exporter // List of exporters for structs with unexported fields
opts Options // List of all fundamental and filter options
}
func newState(opts []Option) *state {
// Always ensure a validator option exists to validate the inputs.
s := &state{opts: Options{validator{}}}
s.curPtrs.Init()
s.processOption(Options(opts))
return s
}
func (s *state) processOption(opt Option) {
switch opt := opt.(type) {
case nil:
case Options:
for _, o := range opt {
s.processOption(o)
}
case coreOption:
type filtered interface {
isFiltered() bool
}
if fopt, ok := opt.(filtered); ok && !fopt.isFiltered() {
panic(fmt.Sprintf("cannot use an unfiltered option: %v", opt))
}
s.opts = append(s.opts, opt)
case exporter:
s.exporters = append(s.exporters, opt)
case reporter:
s.reporters = append(s.reporters, opt)
default:
panic(fmt.Sprintf("unknown option %T", opt))
}
}
// statelessCompare compares two values and returns the result.
// This function is stateless in that it does not alter the current result,
// or output to any registered reporters.
func (s *state) statelessCompare(step PathStep) diff.Result {
// We do not save and restore curPath and curPtrs because all of the
// compareX methods should properly push and pop from them.
// It is an implementation bug if the contents of the paths differ from
// when calling this function to when returning from it.
oldResult, oldReporters := s.result, s.reporters
s.result = diff.Result{} // Reset result
s.reporters = nil // Remove reporters to avoid spurious printouts
s.compareAny(step)
res := s.result
s.result, s.reporters = oldResult, oldReporters
return res
}
func (s *state) compareAny(step PathStep) {
// Update the path stack.
s.curPath.push(step)
defer s.curPath.pop()
for _, r := range s.reporters {
r.PushStep(step)
defer r.PopStep()
}
s.recChecker.Check(s.curPath)
// Cycle-detection for slice elements (see NOTE in compareSlice).
t := step.Type()
vx, vy := step.Values()
if si, ok := step.(SliceIndex); ok && si.isSlice && vx.IsValid() && vy.IsValid() {
px, py := vx.Addr(), vy.Addr()
if eq, visited := s.curPtrs.Push(px, py); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(px, py)
}
// Rule 1: Check whether an option applies on this node in the value tree.
if s.tryOptions(t, vx, vy) {
return
}
// Rule 2: Check whether the type has a valid Equal method.
if s.tryMethod(t, vx, vy) {
return
}
// Rule 3: Compare based on the underlying kind.
switch t.Kind() {
case reflect.Bool:
s.report(vx.Bool() == vy.Bool(), 0)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
s.report(vx.Int() == vy.Int(), 0)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
s.report(vx.Uint() == vy.Uint(), 0)
case reflect.Float32, reflect.Float64:
s.report(vx.Float() == vy.Float(), 0)
case reflect.Complex64, reflect.Complex128:
s.report(vx.Complex() == vy.Complex(), 0)
case reflect.String:
s.report(vx.String() == vy.String(), 0)
case reflect.Chan, reflect.UnsafePointer:
s.report(vx.Pointer() == vy.Pointer(), 0)
case reflect.Func:
s.report(vx.IsNil() && vy.IsNil(), 0)
case reflect.Struct:
s.compareStruct(t, vx, vy)
case reflect.Slice, reflect.Array:
s.compareSlice(t, vx, vy)
case reflect.Map:
s.compareMap(t, vx, vy)
case reflect.Ptr:
s.comparePtr(t, vx, vy)
case reflect.Interface:
s.compareInterface(t, vx, vy)
default:
panic(fmt.Sprintf("%v kind not handled", t.Kind()))
}
}
func (s *state) tryOptions(t reflect.Type, vx, vy reflect.Value) bool {
// Evaluate all filters and apply the remaining options.
if opt := s.opts.filter(s, t, vx, vy); opt != nil {
opt.apply(s, vx, vy)
return true
}
return false
}
func (s *state) tryMethod(t reflect.Type, vx, vy reflect.Value) bool {
// Check if this type even has an Equal method.
m, ok := t.MethodByName("Equal")
if !ok || !function.IsType(m.Type, function.EqualAssignable) {
return false
}
eq := s.callTTBFunc(m.Func, vx, vy)
s.report(eq, reportByMethod)
return true
}
func (s *state) callTRFunc(f, v reflect.Value, step Transform) reflect.Value {
if !s.dynChecker.Next() {
return f.Call([]reflect.Value{v})[0]
}
// Run the function twice and ensure that we get the same results back.
// We run in goroutines so that the race detector (if enabled) can detect
// unsafe mutations to the input.
c := make(chan reflect.Value)
go detectRaces(c, f, v)
got := <-c
want := f.Call([]reflect.Value{v})[0]
if step.vx, step.vy = got, want; !s.statelessCompare(step).Equal() {
// To avoid false-positives with non-reflexive equality operations,
// we sanity check whether a value is equal to itself.
if step.vx, step.vy = want, want; !s.statelessCompare(step).Equal() {
return want
}
panic(fmt.Sprintf("non-deterministic function detected: %s", function.NameOf(f)))
}
return want
}
func (s *state) callTTBFunc(f, x, y reflect.Value) bool {
if !s.dynChecker.Next() {
return f.Call([]reflect.Value{x, y})[0].Bool()
}
// Swapping the input arguments is sufficient to check that
// f is symmetric and deterministic.
// We run in goroutines so that the race detector (if enabled) can detect
// unsafe mutations to the input.
c := make(chan reflect.Value)
go detectRaces(c, f, y, x)
got := <-c
want := f.Call([]reflect.Value{x, y})[0].Bool()
if !got.IsValid() || got.Bool() != want {
panic(fmt.Sprintf("non-deterministic or non-symmetric function detected: %s", function.NameOf(f)))
}
return want
}
func detectRaces(c chan<- reflect.Value, f reflect.Value, vs ...reflect.Value) {
var ret reflect.Value
defer func() {
recover() // Ignore panics, let the other call to f panic instead
c <- ret
}()
ret = f.Call(vs)[0]
}
func (s *state) compareStruct(t reflect.Type, vx, vy reflect.Value) {
var addr bool
var vax, vay reflect.Value // Addressable versions of vx and vy
var mayForce, mayForceInit bool
step := StructField{&structField{}}
for i := 0; i < t.NumField(); i++ {
step.typ = t.Field(i).Type
step.vx = vx.Field(i)
step.vy = vy.Field(i)
step.name = t.Field(i).Name
step.idx = i
step.unexported = !isExported(step.name)
if step.unexported {
if step.name == "_" {
continue
}
// Defer checking of unexported fields until later to give an
// Ignore a chance to ignore the field.
if !vax.IsValid() || !vay.IsValid() {
// For retrieveUnexportedField to work, the parent struct must
// be addressable. Create a new copy of the values if
// necessary to make them addressable.
addr = vx.CanAddr() || vy.CanAddr()
vax = makeAddressable(vx)
vay = makeAddressable(vy)
}
if !mayForceInit {
for _, xf := range s.exporters {
mayForce = mayForce || xf(t)
}
mayForceInit = true
}
step.mayForce = mayForce
step.paddr = addr
step.pvx = vax
step.pvy = vay
step.field = t.Field(i)
}
s.compareAny(step)
}
}
func (s *state) compareSlice(t reflect.Type, vx, vy reflect.Value) {
isSlice := t.Kind() == reflect.Slice
if isSlice && (vx.IsNil() || vy.IsNil()) {
s.report(vx.IsNil() && vy.IsNil(), 0)
return
}
// NOTE: It is incorrect to call curPtrs.Push on the slice header pointer
// since slices represents a list of pointers, rather than a single pointer.
// The pointer checking logic must be handled on a per-element basis
// in compareAny.
//
// A slice header (see reflect.SliceHeader) in Go is a tuple of a starting
// pointer P, a length N, and a capacity C. Supposing each slice element has
// a memory size of M, then the slice is equivalent to the list of pointers:
// [P+i*M for i in range(N)]
//
// For example, v[:0] and v[:1] are slices with the same starting pointer,
// but they are clearly different values. Using the slice pointer alone
// violates the assumption that equal pointers implies equal values.
step := SliceIndex{&sliceIndex{pathStep: pathStep{typ: t.Elem()}, isSlice: isSlice}}
withIndexes := func(ix, iy int) SliceIndex {
if ix >= 0 {
step.vx, step.xkey = vx.Index(ix), ix
} else {
step.vx, step.xkey = reflect.Value{}, -1
}
if iy >= 0 {
step.vy, step.ykey = vy.Index(iy), iy
} else {
step.vy, step.ykey = reflect.Value{}, -1
}
return step
}
// Ignore options are able to ignore missing elements in a slice.
// However, detecting these reliably requires an optimal differencing
// algorithm, for which diff.Difference is not.
//
// Instead, we first iterate through both slices to detect which elements
// would be ignored if standing alone. The index of non-discarded elements
// are stored in a separate slice, which diffing is then performed on.
var indexesX, indexesY []int
var ignoredX, ignoredY []bool
for ix := 0; ix < vx.Len(); ix++ {
ignored := s.statelessCompare(withIndexes(ix, -1)).NumDiff == 0
if !ignored {
indexesX = append(indexesX, ix)
}
ignoredX = append(ignoredX, ignored)
}
for iy := 0; iy < vy.Len(); iy++ {
ignored := s.statelessCompare(withIndexes(-1, iy)).NumDiff == 0
if !ignored {
indexesY = append(indexesY, iy)
}
ignoredY = append(ignoredY, ignored)
}
// Compute an edit-script for slices vx and vy (excluding ignored elements).
edits := diff.Difference(len(indexesX), len(indexesY), func(ix, iy int) diff.Result {
return s.statelessCompare(withIndexes(indexesX[ix], indexesY[iy]))
})
// Replay the ignore-scripts and the edit-script.
var ix, iy int
for ix < vx.Len() || iy < vy.Len() {
var e diff.EditType
switch {
case ix < len(ignoredX) && ignoredX[ix]:
e = diff.UniqueX
case iy < len(ignoredY) && ignoredY[iy]:
e = diff.UniqueY
default:
e, edits = edits[0], edits[1:]
}
switch e {
case diff.UniqueX:
s.compareAny(withIndexes(ix, -1))
ix++
case diff.UniqueY:
s.compareAny(withIndexes(-1, iy))
iy++
default:
s.compareAny(withIndexes(ix, iy))
ix++
iy++
}
}
}
func (s *state) compareMap(t reflect.Type, vx, vy reflect.Value) {
if vx.IsNil() || vy.IsNil() {
s.report(vx.IsNil() && vy.IsNil(), 0)
return
}
// Cycle-detection for maps.
if eq, visited := s.curPtrs.Push(vx, vy); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(vx, vy)
// We combine and sort the two map keys so that we can perform the
// comparisons in a deterministic order.
step := MapIndex{&mapIndex{pathStep: pathStep{typ: t.Elem()}}}
for _, k := range value.SortKeys(append(vx.MapKeys(), vy.MapKeys()...)) {
step.vx = vx.MapIndex(k)
step.vy = vy.MapIndex(k)
step.key = k
if !step.vx.IsValid() && !step.vy.IsValid() {
// It is possible for both vx and vy to be invalid if the
// key contained a NaN value in it.
//
// Even with the ability to retrieve NaN keys in Go 1.12,
// there still isn't a sensible way to compare the values since
// a NaN key may map to multiple unordered values.
// The most reasonable way to compare NaNs would be to compare the
// set of values. However, this is impossible to do efficiently
// since set equality is provably an O(n^2) operation given only
// an Equal function. If we had a Less function or Hash function,
// this could be done in O(n*log(n)) or O(n), respectively.
//
// Rather than adding complex logic to deal with NaNs, make it
// the user's responsibility to compare such obscure maps.
const help = "consider providing a Comparer to compare the map"
panic(fmt.Sprintf("%#v has map key with NaNs\n%s", s.curPath, help))
}
s.compareAny(step)
}
}
func (s *state) comparePtr(t reflect.Type, vx, vy reflect.Value) {
if vx.IsNil() || vy.IsNil() {
s.report(vx.IsNil() && vy.IsNil(), 0)
return
}
// Cycle-detection for pointers.
if eq, visited := s.curPtrs.Push(vx, vy); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(vx, vy)
vx, vy = vx.Elem(), vy.Elem()
s.compareAny(Indirect{&indirect{pathStep{t.Elem(), vx, vy}}})
}
func (s *state) compareInterface(t reflect.Type, vx, vy reflect.Value) {
if vx.IsNil() || vy.IsNil() {
s.report(vx.IsNil() && vy.IsNil(), 0)
return
}
vx, vy = vx.Elem(), vy.Elem()
if vx.Type() != vy.Type() {
s.report(false, 0)
return
}
s.compareAny(TypeAssertion{&typeAssertion{pathStep{vx.Type(), vx, vy}}})
}
func (s *state) report(eq bool, rf resultFlags) {
if rf&reportByIgnore == 0 {
if eq {
s.result.NumSame++
rf |= reportEqual
} else {
s.result.NumDiff++
rf |= reportUnequal
}
}
for _, r := range s.reporters {
r.Report(Result{flags: rf})
}
}
// recChecker tracks the state needed to periodically perform checks that
// user provided transformers are not stuck in an infinitely recursive cycle.
type recChecker struct{ next int }
// Check scans the Path for any recursive transformers and panics when any
// recursive transformers are detected. Note that the presence of a
// recursive Transformer does not necessarily imply an infinite cycle.
// As such, this check only activates after some minimal number of path steps.
func (rc *recChecker) Check(p Path) {
const minLen = 1 << 16
if rc.next == 0 {
rc.next = minLen
}
if len(p) < rc.next {
return
}
rc.next <<= 1
// Check whether the same transformer has appeared at least twice.
var ss []string
m := map[Option]int{}
for _, ps := range p {
if t, ok := ps.(Transform); ok {
t := t.Option()
if m[t] == 1 { // Transformer was used exactly once before
tf := t.(*transformer).fnc.Type()
ss = append(ss, fmt.Sprintf("%v: %v => %v", t, tf.In(0), tf.Out(0)))
}
m[t]++
}
}
if len(ss) > 0 {
const warning = "recursive set of Transformers detected"
const help = "consider using cmpopts.AcyclicTransformer"
set := strings.Join(ss, "\n\t")
panic(fmt.Sprintf("%s:\n\t%s\n%s", warning, set, help))
}
}
// dynChecker tracks the state needed to periodically perform checks that
// user provided functions are symmetric and deterministic.
// The zero value is safe for immediate use.
type dynChecker struct{ curr, next int }
// Next increments the state and reports whether a check should be performed.
//
// Checks occur every Nth function call, where N is a triangular number:
//
// 0 1 3 6 10 15 21 28 36 45 55 66 78 91 105 120 136 153 171 190 ...
//
// See https://en.wikipedia.org/wiki/Triangular_number
//
// This sequence ensures that the cost of checks drops significantly as
// the number of functions calls grows larger.
func (dc *dynChecker) Next() bool {
ok := dc.curr == dc.next
if ok {
dc.curr = 0
dc.next++
}
dc.curr++
return ok
}
// makeAddressable returns a value that is always addressable.
// It returns the input verbatim if it is already addressable,
// otherwise it creates a new value and returns an addressable copy.
func makeAddressable(v reflect.Value) reflect.Value {
if v.CanAddr() {
return v
}
vc := reflect.New(v.Type()).Elem()
vc.Set(v)
return vc
}

View File

@@ -0,0 +1,31 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"reflect"
"unsafe"
)
// retrieveUnexportedField uses unsafe to forcibly retrieve any field from
// a struct such that the value has read-write permissions.
//
// The parent struct, v, must be addressable, while f must be a StructField
// describing the field to retrieve. If addr is false,
// then the returned value will be shallowed copied to be non-addressable.
func retrieveUnexportedField(v reflect.Value, f reflect.StructField, addr bool) reflect.Value {
ve := reflect.NewAt(f.Type, unsafe.Pointer(uintptr(unsafe.Pointer(v.UnsafeAddr()))+f.Offset)).Elem()
if !addr {
// A field is addressable if and only if the struct is addressable.
// If the original parent value was not addressable, shallow copy the
// value to make it non-addressable to avoid leaking an implementation
// detail of how forcibly exporting a field works.
if ve.Kind() == reflect.Interface && ve.IsNil() {
return reflect.Zero(f.Type)
}
return reflect.ValueOf(ve.Interface()).Convert(f.Type)
}
return ve
}

View File

@@ -0,0 +1,18 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !cmp_debug
// +build !cmp_debug
package diff
var debug debugger
type debugger struct{}
func (debugger) Begin(_, _ int, f EqualFunc, _, _ *EditScript) EqualFunc {
return f
}
func (debugger) Update() {}
func (debugger) Finish() {}

View File

@@ -0,0 +1,123 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build cmp_debug
// +build cmp_debug
package diff
import (
"fmt"
"strings"
"sync"
"time"
)
// The algorithm can be seen running in real-time by enabling debugging:
// go test -tags=cmp_debug -v
//
// Example output:
// === RUN TestDifference/#34
// ┌───────────────────────────────┐
// │ \ · · · · · · · · · · · · · · │
// │ · # · · · · · · · · · · · · · │
// │ · \ · · · · · · · · · · · · · │
// │ · · \ · · · · · · · · · · · · │
// │ · · · X # · · · · · · · · · · │
// │ · · · # \ · · · · · · · · · · │
// │ · · · · · # # · · · · · · · · │
// │ · · · · · # \ · · · · · · · · │
// │ · · · · · · · \ · · · · · · · │
// │ · · · · · · · · \ · · · · · · │
// │ · · · · · · · · · \ · · · · · │
// │ · · · · · · · · · · \ · · # · │
// │ · · · · · · · · · · · \ # # · │
// │ · · · · · · · · · · · # # # · │
// │ · · · · · · · · · · # # # # · │
// │ · · · · · · · · · # # # # # · │
// │ · · · · · · · · · · · · · · \ │
// └───────────────────────────────┘
// [.Y..M.XY......YXYXY.|]
//
// The grid represents the edit-graph where the horizontal axis represents
// list X and the vertical axis represents list Y. The start of the two lists
// is the top-left, while the ends are the bottom-right. The '·' represents
// an unexplored node in the graph. The '\' indicates that the two symbols
// from list X and Y are equal. The 'X' indicates that two symbols are similar
// (but not exactly equal) to each other. The '#' indicates that the two symbols
// are different (and not similar). The algorithm traverses this graph trying to
// make the paths starting in the top-left and the bottom-right connect.
//
// The series of '.', 'X', 'Y', and 'M' characters at the bottom represents
// the currently established path from the forward and reverse searches,
// separated by a '|' character.
const (
updateDelay = 100 * time.Millisecond
finishDelay = 500 * time.Millisecond
ansiTerminal = true // ANSI escape codes used to move terminal cursor
)
var debug debugger
type debugger struct {
sync.Mutex
p1, p2 EditScript
fwdPath, revPath *EditScript
grid []byte
lines int
}
func (dbg *debugger) Begin(nx, ny int, f EqualFunc, p1, p2 *EditScript) EqualFunc {
dbg.Lock()
dbg.fwdPath, dbg.revPath = p1, p2
top := "┌─" + strings.Repeat("──", nx) + "┐\n"
row := "│ " + strings.Repeat("· ", nx) + "│\n"
btm := "└─" + strings.Repeat("──", nx) + "┘\n"
dbg.grid = []byte(top + strings.Repeat(row, ny) + btm)
dbg.lines = strings.Count(dbg.String(), "\n")
fmt.Print(dbg)
// Wrap the EqualFunc so that we can intercept each result.
return func(ix, iy int) (r Result) {
cell := dbg.grid[len(top)+iy*len(row):][len("│ ")+len("· ")*ix:][:len("·")]
for i := range cell {
cell[i] = 0 // Zero out the multiple bytes of UTF-8 middle-dot
}
switch r = f(ix, iy); {
case r.Equal():
cell[0] = '\\'
case r.Similar():
cell[0] = 'X'
default:
cell[0] = '#'
}
return
}
}
func (dbg *debugger) Update() {
dbg.print(updateDelay)
}
func (dbg *debugger) Finish() {
dbg.print(finishDelay)
dbg.Unlock()
}
func (dbg *debugger) String() string {
dbg.p1, dbg.p2 = *dbg.fwdPath, dbg.p2[:0]
for i := len(*dbg.revPath) - 1; i >= 0; i-- {
dbg.p2 = append(dbg.p2, (*dbg.revPath)[i])
}
return fmt.Sprintf("%s[%v|%v]\n\n", dbg.grid, dbg.p1, dbg.p2)
}
func (dbg *debugger) print(d time.Duration) {
if ansiTerminal {
fmt.Printf("\x1b[%dA", dbg.lines) // Reset terminal cursor
}
fmt.Print(dbg)
time.Sleep(d)
}

View File

@@ -0,0 +1,402 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package diff implements an algorithm for producing edit-scripts.
// The edit-script is a sequence of operations needed to transform one list
// of symbols into another (or vice-versa). The edits allowed are insertions,
// deletions, and modifications. The summation of all edits is called the
// Levenshtein distance as this problem is well-known in computer science.
//
// This package prioritizes performance over accuracy. That is, the run time
// is more important than obtaining a minimal Levenshtein distance.
package diff
import (
"math/rand"
"time"
"github.com/google/go-cmp/cmp/internal/flags"
)
// EditType represents a single operation within an edit-script.
type EditType uint8
const (
// Identity indicates that a symbol pair is identical in both list X and Y.
Identity EditType = iota
// UniqueX indicates that a symbol only exists in X and not Y.
UniqueX
// UniqueY indicates that a symbol only exists in Y and not X.
UniqueY
// Modified indicates that a symbol pair is a modification of each other.
Modified
)
// EditScript represents the series of differences between two lists.
type EditScript []EditType
// String returns a human-readable string representing the edit-script where
// Identity, UniqueX, UniqueY, and Modified are represented by the
// '.', 'X', 'Y', and 'M' characters, respectively.
func (es EditScript) String() string {
b := make([]byte, len(es))
for i, e := range es {
switch e {
case Identity:
b[i] = '.'
case UniqueX:
b[i] = 'X'
case UniqueY:
b[i] = 'Y'
case Modified:
b[i] = 'M'
default:
panic("invalid edit-type")
}
}
return string(b)
}
// stats returns a histogram of the number of each type of edit operation.
func (es EditScript) stats() (s struct{ NI, NX, NY, NM int }) {
for _, e := range es {
switch e {
case Identity:
s.NI++
case UniqueX:
s.NX++
case UniqueY:
s.NY++
case Modified:
s.NM++
default:
panic("invalid edit-type")
}
}
return
}
// Dist is the Levenshtein distance and is guaranteed to be 0 if and only if
// lists X and Y are equal.
func (es EditScript) Dist() int { return len(es) - es.stats().NI }
// LenX is the length of the X list.
func (es EditScript) LenX() int { return len(es) - es.stats().NY }
// LenY is the length of the Y list.
func (es EditScript) LenY() int { return len(es) - es.stats().NX }
// EqualFunc reports whether the symbols at indexes ix and iy are equal.
// When called by Difference, the index is guaranteed to be within nx and ny.
type EqualFunc func(ix int, iy int) Result
// Result is the result of comparison.
// NumSame is the number of sub-elements that are equal.
// NumDiff is the number of sub-elements that are not equal.
type Result struct{ NumSame, NumDiff int }
// BoolResult returns a Result that is either Equal or not Equal.
func BoolResult(b bool) Result {
if b {
return Result{NumSame: 1} // Equal, Similar
} else {
return Result{NumDiff: 2} // Not Equal, not Similar
}
}
// Equal indicates whether the symbols are equal. Two symbols are equal
// if and only if NumDiff == 0. If Equal, then they are also Similar.
func (r Result) Equal() bool { return r.NumDiff == 0 }
// Similar indicates whether two symbols are similar and may be represented
// by using the Modified type. As a special case, we consider binary comparisons
// (i.e., those that return Result{1, 0} or Result{0, 1}) to be similar.
//
// The exact ratio of NumSame to NumDiff to determine similarity may change.
func (r Result) Similar() bool {
// Use NumSame+1 to offset NumSame so that binary comparisons are similar.
return r.NumSame+1 >= r.NumDiff
}
var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
// Difference reports whether two lists of lengths nx and ny are equal
// given the definition of equality provided as f.
//
// This function returns an edit-script, which is a sequence of operations
// needed to convert one list into the other. The following invariants for
// the edit-script are maintained:
// - eq == (es.Dist()==0)
// - nx == es.LenX()
// - ny == es.LenY()
//
// This algorithm is not guaranteed to be an optimal solution (i.e., one that
// produces an edit-script with a minimal Levenshtein distance). This algorithm
// favors performance over optimality. The exact output is not guaranteed to
// be stable and may change over time.
func Difference(nx, ny int, f EqualFunc) (es EditScript) {
// This algorithm is based on traversing what is known as an "edit-graph".
// See Figure 1 from "An O(ND) Difference Algorithm and Its Variations"
// by Eugene W. Myers. Since D can be as large as N itself, this is
// effectively O(N^2). Unlike the algorithm from that paper, we are not
// interested in the optimal path, but at least some "decent" path.
//
// For example, let X and Y be lists of symbols:
// X = [A B C A B B A]
// Y = [C B A B A C]
//
// The edit-graph can be drawn as the following:
// A B C A B B A
// ┌─────────────┐
// C │_|_|\|_|_|_|_│ 0
// B │_|\|_|_|\|\|_│ 1
// A │\|_|_|\|_|_|\│ 2
// B │_|\|_|_|\|\|_│ 3
// A │\|_|_|\|_|_|\│ 4
// C │ | |\| | | | │ 5
// └─────────────┘ 6
// 0 1 2 3 4 5 6 7
//
// List X is written along the horizontal axis, while list Y is written
// along the vertical axis. At any point on this grid, if the symbol in
// list X matches the corresponding symbol in list Y, then a '\' is drawn.
// The goal of any minimal edit-script algorithm is to find a path from the
// top-left corner to the bottom-right corner, while traveling through the
// fewest horizontal or vertical edges.
// A horizontal edge is equivalent to inserting a symbol from list X.
// A vertical edge is equivalent to inserting a symbol from list Y.
// A diagonal edge is equivalent to a matching symbol between both X and Y.
// Invariants:
// - 0 ≤ fwdPath.X ≤ (fwdFrontier.X, revFrontier.X) ≤ revPath.X ≤ nx
// - 0 ≤ fwdPath.Y ≤ (fwdFrontier.Y, revFrontier.Y) ≤ revPath.Y ≤ ny
//
// In general:
// - fwdFrontier.X < revFrontier.X
// - fwdFrontier.Y < revFrontier.Y
//
// Unless, it is time for the algorithm to terminate.
fwdPath := path{+1, point{0, 0}, make(EditScript, 0, (nx+ny)/2)}
revPath := path{-1, point{nx, ny}, make(EditScript, 0)}
fwdFrontier := fwdPath.point // Forward search frontier
revFrontier := revPath.point // Reverse search frontier
// Search budget bounds the cost of searching for better paths.
// The longest sequence of non-matching symbols that can be tolerated is
// approximately the square-root of the search budget.
searchBudget := 4 * (nx + ny) // O(n)
// Running the tests with the "cmp_debug" build tag prints a visualization
// of the algorithm running in real-time. This is educational for
// understanding how the algorithm works. See debug_enable.go.
f = debug.Begin(nx, ny, f, &fwdPath.es, &revPath.es)
// The algorithm below is a greedy, meet-in-the-middle algorithm for
// computing sub-optimal edit-scripts between two lists.
//
// The algorithm is approximately as follows:
// - Searching for differences switches back-and-forth between
// a search that starts at the beginning (the top-left corner), and
// a search that starts at the end (the bottom-right corner).
// The goal of the search is connect with the search
// from the opposite corner.
// - As we search, we build a path in a greedy manner,
// where the first match seen is added to the path (this is sub-optimal,
// but provides a decent result in practice). When matches are found,
// we try the next pair of symbols in the lists and follow all matches
// as far as possible.
// - When searching for matches, we search along a diagonal going through
// through the "frontier" point. If no matches are found,
// we advance the frontier towards the opposite corner.
// - This algorithm terminates when either the X coordinates or the
// Y coordinates of the forward and reverse frontier points ever intersect.
// This algorithm is correct even if searching only in the forward direction
// or in the reverse direction. We do both because it is commonly observed
// that two lists commonly differ because elements were added to the front
// or end of the other list.
//
// Non-deterministically start with either the forward or reverse direction
// to introduce some deliberate instability so that we have the flexibility
// to change this algorithm in the future.
if flags.Deterministic || randBool {
goto forwardSearch
} else {
goto reverseSearch
}
forwardSearch:
{
// Forward search from the beginning.
if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
goto finishSearch
}
for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
// Search in a diagonal pattern for a match.
z := zigzag(i)
p := point{fwdFrontier.X + z, fwdFrontier.Y - z}
switch {
case p.X >= revPath.X || p.Y < fwdPath.Y:
stop1 = true // Hit top-right corner
case p.Y >= revPath.Y || p.X < fwdPath.X:
stop2 = true // Hit bottom-left corner
case f(p.X, p.Y).Equal():
// Match found, so connect the path to this point.
fwdPath.connect(p, f)
fwdPath.append(Identity)
// Follow sequence of matches as far as possible.
for fwdPath.X < revPath.X && fwdPath.Y < revPath.Y {
if !f(fwdPath.X, fwdPath.Y).Equal() {
break
}
fwdPath.append(Identity)
}
fwdFrontier = fwdPath.point
stop1, stop2 = true, true
default:
searchBudget-- // Match not found
}
debug.Update()
}
// Advance the frontier towards reverse point.
if revPath.X-fwdFrontier.X >= revPath.Y-fwdFrontier.Y {
fwdFrontier.X++
} else {
fwdFrontier.Y++
}
goto reverseSearch
}
reverseSearch:
{
// Reverse search from the end.
if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
goto finishSearch
}
for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
// Search in a diagonal pattern for a match.
z := zigzag(i)
p := point{revFrontier.X - z, revFrontier.Y + z}
switch {
case fwdPath.X >= p.X || revPath.Y < p.Y:
stop1 = true // Hit bottom-left corner
case fwdPath.Y >= p.Y || revPath.X < p.X:
stop2 = true // Hit top-right corner
case f(p.X-1, p.Y-1).Equal():
// Match found, so connect the path to this point.
revPath.connect(p, f)
revPath.append(Identity)
// Follow sequence of matches as far as possible.
for fwdPath.X < revPath.X && fwdPath.Y < revPath.Y {
if !f(revPath.X-1, revPath.Y-1).Equal() {
break
}
revPath.append(Identity)
}
revFrontier = revPath.point
stop1, stop2 = true, true
default:
searchBudget-- // Match not found
}
debug.Update()
}
// Advance the frontier towards forward point.
if revFrontier.X-fwdPath.X >= revFrontier.Y-fwdPath.Y {
revFrontier.X--
} else {
revFrontier.Y--
}
goto forwardSearch
}
finishSearch:
// Join the forward and reverse paths and then append the reverse path.
fwdPath.connect(revPath.point, f)
for i := len(revPath.es) - 1; i >= 0; i-- {
t := revPath.es[i]
revPath.es = revPath.es[:i]
fwdPath.append(t)
}
debug.Finish()
return fwdPath.es
}
type path struct {
dir int // +1 if forward, -1 if reverse
point // Leading point of the EditScript path
es EditScript
}
// connect appends any necessary Identity, Modified, UniqueX, or UniqueY types
// to the edit-script to connect p.point to dst.
func (p *path) connect(dst point, f EqualFunc) {
if p.dir > 0 {
// Connect in forward direction.
for dst.X > p.X && dst.Y > p.Y {
switch r := f(p.X, p.Y); {
case r.Equal():
p.append(Identity)
case r.Similar():
p.append(Modified)
case dst.X-p.X >= dst.Y-p.Y:
p.append(UniqueX)
default:
p.append(UniqueY)
}
}
for dst.X > p.X {
p.append(UniqueX)
}
for dst.Y > p.Y {
p.append(UniqueY)
}
} else {
// Connect in reverse direction.
for p.X > dst.X && p.Y > dst.Y {
switch r := f(p.X-1, p.Y-1); {
case r.Equal():
p.append(Identity)
case r.Similar():
p.append(Modified)
case p.Y-dst.Y >= p.X-dst.X:
p.append(UniqueY)
default:
p.append(UniqueX)
}
}
for p.X > dst.X {
p.append(UniqueX)
}
for p.Y > dst.Y {
p.append(UniqueY)
}
}
}
func (p *path) append(t EditType) {
p.es = append(p.es, t)
switch t {
case Identity, Modified:
p.add(p.dir, p.dir)
case UniqueX:
p.add(p.dir, 0)
case UniqueY:
p.add(0, p.dir)
}
debug.Update()
}
type point struct{ X, Y int }
func (p *point) add(dx, dy int) { p.X += dx; p.Y += dy }
// zigzag maps a consecutive sequence of integers to a zig-zag sequence.
//
// [0 1 2 3 4 5 ...] => [0 -1 +1 -2 +2 ...]
func zigzag(x int) int {
if x&1 != 0 {
x = ^x
}
return x >> 1
}

View File

@@ -0,0 +1,9 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flags
// Deterministic controls whether the output of Diff should be deterministic.
// This is only used for testing.
var Deterministic bool

View File

@@ -0,0 +1,106 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package function provides functionality for identifying function types.
package function
import (
"reflect"
"regexp"
"runtime"
"strings"
)
type funcType int
const (
_ funcType = iota
tbFunc // func(T) bool
ttbFunc // func(T, T) bool
ttiFunc // func(T, T) int
trbFunc // func(T, R) bool
tibFunc // func(T, I) bool
trFunc // func(T) R
Equal = ttbFunc // func(T, T) bool
EqualAssignable = tibFunc // func(T, I) bool; encapsulates func(T, T) bool
Transformer = trFunc // func(T) R
ValueFilter = ttbFunc // func(T, T) bool
Less = ttbFunc // func(T, T) bool
Compare = ttiFunc // func(T, T) int
ValuePredicate = tbFunc // func(T) bool
KeyValuePredicate = trbFunc // func(T, R) bool
)
var boolType = reflect.TypeOf(true)
var intType = reflect.TypeOf(0)
// IsType reports whether the reflect.Type is of the specified function type.
func IsType(t reflect.Type, ft funcType) bool {
if t == nil || t.Kind() != reflect.Func || t.IsVariadic() {
return false
}
ni, no := t.NumIn(), t.NumOut()
switch ft {
case tbFunc: // func(T) bool
if ni == 1 && no == 1 && t.Out(0) == boolType {
return true
}
case ttbFunc: // func(T, T) bool
if ni == 2 && no == 1 && t.In(0) == t.In(1) && t.Out(0) == boolType {
return true
}
case ttiFunc: // func(T, T) int
if ni == 2 && no == 1 && t.In(0) == t.In(1) && t.Out(0) == intType {
return true
}
case trbFunc: // func(T, R) bool
if ni == 2 && no == 1 && t.Out(0) == boolType {
return true
}
case tibFunc: // func(T, I) bool
if ni == 2 && no == 1 && t.In(0).AssignableTo(t.In(1)) && t.Out(0) == boolType {
return true
}
case trFunc: // func(T) R
if ni == 1 && no == 1 {
return true
}
}
return false
}
var lastIdentRx = regexp.MustCompile(`[_\p{L}][_\p{L}\p{N}]*$`)
// NameOf returns the name of the function value.
func NameOf(v reflect.Value) string {
fnc := runtime.FuncForPC(v.Pointer())
if fnc == nil {
return "<unknown>"
}
fullName := fnc.Name() // e.g., "long/path/name/mypkg.(*MyType).(long/path/name/mypkg.myMethod)-fm"
// Method closures have a "-fm" suffix.
fullName = strings.TrimSuffix(fullName, "-fm")
var name string
for len(fullName) > 0 {
inParen := strings.HasSuffix(fullName, ")")
fullName = strings.TrimSuffix(fullName, ")")
s := lastIdentRx.FindString(fullName)
if s == "" {
break
}
name = s + "." + name
fullName = strings.TrimSuffix(fullName, s)
if i := strings.LastIndexByte(fullName, '('); inParen && i >= 0 {
fullName = fullName[:i]
}
fullName = strings.TrimSuffix(fullName, ".")
}
return strings.TrimSuffix(name, ".")
}

View File

@@ -0,0 +1,164 @@
// Copyright 2020, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package value
import (
"reflect"
"strconv"
)
var anyType = reflect.TypeOf((*interface{})(nil)).Elem()
// TypeString is nearly identical to reflect.Type.String,
// but has an additional option to specify that full type names be used.
func TypeString(t reflect.Type, qualified bool) string {
return string(appendTypeName(nil, t, qualified, false))
}
func appendTypeName(b []byte, t reflect.Type, qualified, elideFunc bool) []byte {
// BUG: Go reflection provides no way to disambiguate two named types
// of the same name and within the same package,
// but declared within the namespace of different functions.
// Use the "any" alias instead of "interface{}" for better readability.
if t == anyType {
return append(b, "any"...)
}
// Named type.
if t.Name() != "" {
if qualified && t.PkgPath() != "" {
b = append(b, '"')
b = append(b, t.PkgPath()...)
b = append(b, '"')
b = append(b, '.')
b = append(b, t.Name()...)
} else {
b = append(b, t.String()...)
}
return b
}
// Unnamed type.
switch k := t.Kind(); k {
case reflect.Bool, reflect.String, reflect.UnsafePointer,
reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
b = append(b, k.String()...)
case reflect.Chan:
if t.ChanDir() == reflect.RecvDir {
b = append(b, "<-"...)
}
b = append(b, "chan"...)
if t.ChanDir() == reflect.SendDir {
b = append(b, "<-"...)
}
b = append(b, ' ')
b = appendTypeName(b, t.Elem(), qualified, false)
case reflect.Func:
if !elideFunc {
b = append(b, "func"...)
}
b = append(b, '(')
for i := 0; i < t.NumIn(); i++ {
if i > 0 {
b = append(b, ", "...)
}
if i == t.NumIn()-1 && t.IsVariadic() {
b = append(b, "..."...)
b = appendTypeName(b, t.In(i).Elem(), qualified, false)
} else {
b = appendTypeName(b, t.In(i), qualified, false)
}
}
b = append(b, ')')
switch t.NumOut() {
case 0:
// Do nothing
case 1:
b = append(b, ' ')
b = appendTypeName(b, t.Out(0), qualified, false)
default:
b = append(b, " ("...)
for i := 0; i < t.NumOut(); i++ {
if i > 0 {
b = append(b, ", "...)
}
b = appendTypeName(b, t.Out(i), qualified, false)
}
b = append(b, ')')
}
case reflect.Struct:
b = append(b, "struct{ "...)
for i := 0; i < t.NumField(); i++ {
if i > 0 {
b = append(b, "; "...)
}
sf := t.Field(i)
if !sf.Anonymous {
if qualified && sf.PkgPath != "" {
b = append(b, '"')
b = append(b, sf.PkgPath...)
b = append(b, '"')
b = append(b, '.')
}
b = append(b, sf.Name...)
b = append(b, ' ')
}
b = appendTypeName(b, sf.Type, qualified, false)
if sf.Tag != "" {
b = append(b, ' ')
b = strconv.AppendQuote(b, string(sf.Tag))
}
}
if b[len(b)-1] == ' ' {
b = b[:len(b)-1]
} else {
b = append(b, ' ')
}
b = append(b, '}')
case reflect.Slice, reflect.Array:
b = append(b, '[')
if k == reflect.Array {
b = strconv.AppendUint(b, uint64(t.Len()), 10)
}
b = append(b, ']')
b = appendTypeName(b, t.Elem(), qualified, false)
case reflect.Map:
b = append(b, "map["...)
b = appendTypeName(b, t.Key(), qualified, false)
b = append(b, ']')
b = appendTypeName(b, t.Elem(), qualified, false)
case reflect.Ptr:
b = append(b, '*')
b = appendTypeName(b, t.Elem(), qualified, false)
case reflect.Interface:
b = append(b, "interface{ "...)
for i := 0; i < t.NumMethod(); i++ {
if i > 0 {
b = append(b, "; "...)
}
m := t.Method(i)
if qualified && m.PkgPath != "" {
b = append(b, '"')
b = append(b, m.PkgPath...)
b = append(b, '"')
b = append(b, '.')
}
b = append(b, m.Name...)
b = appendTypeName(b, m.Type, qualified, true)
}
if b[len(b)-1] == ' ' {
b = b[:len(b)-1]
} else {
b = append(b, ' ')
}
b = append(b, '}')
default:
panic("invalid kind: " + k.String())
}
return b
}

View File

@@ -0,0 +1,34 @@
// Copyright 2018, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package value
import (
"reflect"
"unsafe"
)
// Pointer is an opaque typed pointer and is guaranteed to be comparable.
type Pointer struct {
p unsafe.Pointer
t reflect.Type
}
// PointerOf returns a Pointer from v, which must be a
// reflect.Ptr, reflect.Slice, or reflect.Map.
func PointerOf(v reflect.Value) Pointer {
// The proper representation of a pointer is unsafe.Pointer,
// which is necessary if the GC ever uses a moving collector.
return Pointer{unsafe.Pointer(v.Pointer()), v.Type()}
}
// IsNil reports whether the pointer is nil.
func (p Pointer) IsNil() bool {
return p.p == nil
}
// Uintptr returns the pointer as a uintptr.
func (p Pointer) Uintptr() uintptr {
return uintptr(p.p)
}

View File

@@ -0,0 +1,106 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package value
import (
"fmt"
"math"
"reflect"
"sort"
)
// SortKeys sorts a list of map keys, deduplicating keys if necessary.
// The type of each value must be comparable.
func SortKeys(vs []reflect.Value) []reflect.Value {
if len(vs) == 0 {
return vs
}
// Sort the map keys.
sort.SliceStable(vs, func(i, j int) bool { return isLess(vs[i], vs[j]) })
// Deduplicate keys (fails for NaNs).
vs2 := vs[:1]
for _, v := range vs[1:] {
if isLess(vs2[len(vs2)-1], v) {
vs2 = append(vs2, v)
}
}
return vs2
}
// isLess is a generic function for sorting arbitrary map keys.
// The inputs must be of the same type and must be comparable.
func isLess(x, y reflect.Value) bool {
switch x.Type().Kind() {
case reflect.Bool:
return !x.Bool() && y.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return x.Int() < y.Int()
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return x.Uint() < y.Uint()
case reflect.Float32, reflect.Float64:
// NOTE: This does not sort -0 as less than +0
// since Go maps treat -0 and +0 as equal keys.
fx, fy := x.Float(), y.Float()
return fx < fy || math.IsNaN(fx) && !math.IsNaN(fy)
case reflect.Complex64, reflect.Complex128:
cx, cy := x.Complex(), y.Complex()
rx, ix, ry, iy := real(cx), imag(cx), real(cy), imag(cy)
if rx == ry || (math.IsNaN(rx) && math.IsNaN(ry)) {
return ix < iy || math.IsNaN(ix) && !math.IsNaN(iy)
}
return rx < ry || math.IsNaN(rx) && !math.IsNaN(ry)
case reflect.Ptr, reflect.UnsafePointer, reflect.Chan:
return x.Pointer() < y.Pointer()
case reflect.String:
return x.String() < y.String()
case reflect.Array:
for i := 0; i < x.Len(); i++ {
if isLess(x.Index(i), y.Index(i)) {
return true
}
if isLess(y.Index(i), x.Index(i)) {
return false
}
}
return false
case reflect.Struct:
for i := 0; i < x.NumField(); i++ {
if isLess(x.Field(i), y.Field(i)) {
return true
}
if isLess(y.Field(i), x.Field(i)) {
return false
}
}
return false
case reflect.Interface:
vx, vy := x.Elem(), y.Elem()
if !vx.IsValid() || !vy.IsValid() {
return !vx.IsValid() && vy.IsValid()
}
tx, ty := vx.Type(), vy.Type()
if tx == ty {
return isLess(x.Elem(), y.Elem())
}
if tx.Kind() != ty.Kind() {
return vx.Kind() < vy.Kind()
}
if tx.String() != ty.String() {
return tx.String() < ty.String()
}
if tx.PkgPath() != ty.PkgPath() {
return tx.PkgPath() < ty.PkgPath()
}
// This can happen in rare situations, so we fallback to just comparing
// the unique pointer for a reflect.Type. This guarantees deterministic
// ordering within a program, but it is obviously not stable.
return reflect.ValueOf(vx.Type()).Pointer() < reflect.ValueOf(vy.Type()).Pointer()
default:
// Must be Func, Map, or Slice; which are not comparable.
panic(fmt.Sprintf("%T is not comparable", x.Type()))
}
}

View File

@@ -0,0 +1,562 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"fmt"
"reflect"
"regexp"
"strings"
"github.com/google/go-cmp/cmp/internal/function"
)
// Option configures for specific behavior of [Equal] and [Diff]. In particular,
// the fundamental Option functions ([Ignore], [Transformer], and [Comparer]),
// configure how equality is determined.
//
// The fundamental options may be composed with filters ([FilterPath] and
// [FilterValues]) to control the scope over which they are applied.
//
// The [github.com/google/go-cmp/cmp/cmpopts] package provides helper functions
// for creating options that may be used with [Equal] and [Diff].
type Option interface {
// filter applies all filters and returns the option that remains.
// Each option may only read s.curPath and call s.callTTBFunc.
//
// An Options is returned only if multiple comparers or transformers
// can apply simultaneously and will only contain values of those types
// or sub-Options containing values of those types.
filter(s *state, t reflect.Type, vx, vy reflect.Value) applicableOption
}
// applicableOption represents the following types:
//
// Fundamental: ignore | validator | *comparer | *transformer
// Grouping: Options
type applicableOption interface {
Option
// apply executes the option, which may mutate s or panic.
apply(s *state, vx, vy reflect.Value)
}
// coreOption represents the following types:
//
// Fundamental: ignore | validator | *comparer | *transformer
// Filters: *pathFilter | *valuesFilter
type coreOption interface {
Option
isCore()
}
type core struct{}
func (core) isCore() {}
// Options is a list of [Option] values that also satisfies the [Option] interface.
// Helper comparison packages may return an Options value when packing multiple
// [Option] values into a single [Option]. When this package processes an Options,
// it will be implicitly expanded into a flat list.
//
// Applying a filter on an Options is equivalent to applying that same filter
// on all individual options held within.
type Options []Option
func (opts Options) filter(s *state, t reflect.Type, vx, vy reflect.Value) (out applicableOption) {
for _, opt := range opts {
switch opt := opt.filter(s, t, vx, vy); opt.(type) {
case ignore:
return ignore{} // Only ignore can short-circuit evaluation
case validator:
out = validator{} // Takes precedence over comparer or transformer
case *comparer, *transformer, Options:
switch out.(type) {
case nil:
out = opt
case validator:
// Keep validator
case *comparer, *transformer, Options:
out = Options{out, opt} // Conflicting comparers or transformers
}
}
}
return out
}
func (opts Options) apply(s *state, _, _ reflect.Value) {
const warning = "ambiguous set of applicable options"
const help = "consider using filters to ensure at most one Comparer or Transformer may apply"
var ss []string
for _, opt := range flattenOptions(nil, opts) {
ss = append(ss, fmt.Sprint(opt))
}
set := strings.Join(ss, "\n\t")
panic(fmt.Sprintf("%s at %#v:\n\t%s\n%s", warning, s.curPath, set, help))
}
func (opts Options) String() string {
var ss []string
for _, opt := range opts {
ss = append(ss, fmt.Sprint(opt))
}
return fmt.Sprintf("Options{%s}", strings.Join(ss, ", "))
}
// FilterPath returns a new [Option] where opt is only evaluated if filter f
// returns true for the current [Path] in the value tree.
//
// This filter is called even if a slice element or map entry is missing and
// provides an opportunity to ignore such cases. The filter function must be
// symmetric such that the filter result is identical regardless of whether the
// missing value is from x or y.
//
// The option passed in may be an [Ignore], [Transformer], [Comparer], [Options], or
// a previously filtered [Option].
func FilterPath(f func(Path) bool, opt Option) Option {
if f == nil {
panic("invalid path filter function")
}
if opt := normalizeOption(opt); opt != nil {
return &pathFilter{fnc: f, opt: opt}
}
return nil
}
type pathFilter struct {
core
fnc func(Path) bool
opt Option
}
func (f pathFilter) filter(s *state, t reflect.Type, vx, vy reflect.Value) applicableOption {
if f.fnc(s.curPath) {
return f.opt.filter(s, t, vx, vy)
}
return nil
}
func (f pathFilter) String() string {
return fmt.Sprintf("FilterPath(%s, %v)", function.NameOf(reflect.ValueOf(f.fnc)), f.opt)
}
// FilterValues returns a new [Option] where opt is only evaluated if filter f,
// which is a function of the form "func(T, T) bool", returns true for the
// current pair of values being compared. If either value is invalid or
// the type of the values is not assignable to T, then this filter implicitly
// returns false.
//
// The filter function must be
// symmetric (i.e., agnostic to the order of the inputs) and
// deterministic (i.e., produces the same result when given the same inputs).
// If T is an interface, it is possible that f is called with two values with
// different concrete types that both implement T.
//
// The option passed in may be an [Ignore], [Transformer], [Comparer], [Options], or
// a previously filtered [Option].
func FilterValues(f interface{}, opt Option) Option {
v := reflect.ValueOf(f)
if !function.IsType(v.Type(), function.ValueFilter) || v.IsNil() {
panic(fmt.Sprintf("invalid values filter function: %T", f))
}
if opt := normalizeOption(opt); opt != nil {
vf := &valuesFilter{fnc: v, opt: opt}
if ti := v.Type().In(0); ti.Kind() != reflect.Interface || ti.NumMethod() > 0 {
vf.typ = ti
}
return vf
}
return nil
}
type valuesFilter struct {
core
typ reflect.Type // T
fnc reflect.Value // func(T, T) bool
opt Option
}
func (f valuesFilter) filter(s *state, t reflect.Type, vx, vy reflect.Value) applicableOption {
if !vx.IsValid() || !vx.CanInterface() || !vy.IsValid() || !vy.CanInterface() {
return nil
}
if (f.typ == nil || t.AssignableTo(f.typ)) && s.callTTBFunc(f.fnc, vx, vy) {
return f.opt.filter(s, t, vx, vy)
}
return nil
}
func (f valuesFilter) String() string {
return fmt.Sprintf("FilterValues(%s, %v)", function.NameOf(f.fnc), f.opt)
}
// Ignore is an [Option] that causes all comparisons to be ignored.
// This value is intended to be combined with [FilterPath] or [FilterValues].
// It is an error to pass an unfiltered Ignore option to [Equal].
func Ignore() Option { return ignore{} }
type ignore struct{ core }
func (ignore) isFiltered() bool { return false }
func (ignore) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption { return ignore{} }
func (ignore) apply(s *state, _, _ reflect.Value) { s.report(true, reportByIgnore) }
func (ignore) String() string { return "Ignore()" }
// validator is a sentinel Option type to indicate that some options could not
// be evaluated due to unexported fields, missing slice elements, or
// missing map entries. Both values are validator only for unexported fields.
type validator struct{ core }
func (validator) filter(_ *state, _ reflect.Type, vx, vy reflect.Value) applicableOption {
if !vx.IsValid() || !vy.IsValid() {
return validator{}
}
if !vx.CanInterface() || !vy.CanInterface() {
return validator{}
}
return nil
}
func (validator) apply(s *state, vx, vy reflect.Value) {
// Implies missing slice element or map entry.
if !vx.IsValid() || !vy.IsValid() {
s.report(vx.IsValid() == vy.IsValid(), 0)
return
}
// Unable to Interface implies unexported field without visibility access.
if !vx.CanInterface() || !vy.CanInterface() {
help := "consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported"
var name string
if t := s.curPath.Index(-2).Type(); t.Name() != "" {
// Named type with unexported fields.
name = fmt.Sprintf("%q.%v", t.PkgPath(), t.Name()) // e.g., "path/to/package".MyType
isProtoMessage := func(t reflect.Type) bool {
m, ok := reflect.PointerTo(t).MethodByName("ProtoReflect")
return ok && m.Type.NumIn() == 1 && m.Type.NumOut() == 1 &&
m.Type.Out(0).PkgPath() == "google.golang.org/protobuf/reflect/protoreflect" &&
m.Type.Out(0).Name() == "Message"
}
if isProtoMessage(t) {
help = `consider using "google.golang.org/protobuf/testing/protocmp".Transform to compare proto.Message types`
} else if _, ok := reflect.New(t).Interface().(error); ok {
help = "consider using cmpopts.EquateErrors to compare error values"
} else if t.Comparable() {
help = "consider using cmpopts.EquateComparable to compare comparable Go types"
}
} else {
// Unnamed type with unexported fields. Derive PkgPath from field.
var pkgPath string
for i := 0; i < t.NumField() && pkgPath == ""; i++ {
pkgPath = t.Field(i).PkgPath
}
name = fmt.Sprintf("%q.(%v)", pkgPath, t.String()) // e.g., "path/to/package".(struct { a int })
}
panic(fmt.Sprintf("cannot handle unexported field at %#v:\n\t%v\n%s", s.curPath, name, help))
}
panic("not reachable")
}
// identRx represents a valid identifier according to the Go specification.
const identRx = `[_\p{L}][_\p{L}\p{N}]*`
var identsRx = regexp.MustCompile(`^` + identRx + `(\.` + identRx + `)*$`)
// Transformer returns an [Option] that applies a transformation function that
// converts values of a certain type into that of another.
//
// The transformer f must be a function "func(T) R" that converts values of
// type T to those of type R and is implicitly filtered to input values
// assignable to T. The transformer must not mutate T in any way.
//
// To help prevent some cases of infinite recursive cycles applying the
// same transform to the output of itself (e.g., in the case where the
// input and output types are the same), an implicit filter is added such that
// a transformer is applicable only if that exact transformer is not already
// in the tail of the [Path] since the last non-[Transform] step.
// For situations where the implicit filter is still insufficient,
// consider using [github.com/google/go-cmp/cmp/cmpopts.AcyclicTransformer],
// which adds a filter to prevent the transformer from
// being recursively applied upon itself.
//
// The name is a user provided label that is used as the [Transform.Name] in the
// transformation [PathStep] (and eventually shown in the [Diff] output).
// The name must be a valid identifier or qualified identifier in Go syntax.
// If empty, an arbitrary name is used.
func Transformer(name string, f interface{}) Option {
v := reflect.ValueOf(f)
if !function.IsType(v.Type(), function.Transformer) || v.IsNil() {
panic(fmt.Sprintf("invalid transformer function: %T", f))
}
if name == "" {
name = function.NameOf(v)
if !identsRx.MatchString(name) {
name = "λ" // Lambda-symbol as placeholder name
}
} else if !identsRx.MatchString(name) {
panic(fmt.Sprintf("invalid name: %q", name))
}
tr := &transformer{name: name, fnc: reflect.ValueOf(f)}
if ti := v.Type().In(0); ti.Kind() != reflect.Interface || ti.NumMethod() > 0 {
tr.typ = ti
}
return tr
}
type transformer struct {
core
name string
typ reflect.Type // T
fnc reflect.Value // func(T) R
}
func (tr *transformer) isFiltered() bool { return tr.typ != nil }
func (tr *transformer) filter(s *state, t reflect.Type, _, _ reflect.Value) applicableOption {
for i := len(s.curPath) - 1; i >= 0; i-- {
if t, ok := s.curPath[i].(Transform); !ok {
break // Hit most recent non-Transform step
} else if tr == t.trans {
return nil // Cannot directly use same Transform
}
}
if tr.typ == nil || t.AssignableTo(tr.typ) {
return tr
}
return nil
}
func (tr *transformer) apply(s *state, vx, vy reflect.Value) {
step := Transform{&transform{pathStep{typ: tr.fnc.Type().Out(0)}, tr}}
vvx := s.callTRFunc(tr.fnc, vx, step)
vvy := s.callTRFunc(tr.fnc, vy, step)
step.vx, step.vy = vvx, vvy
s.compareAny(step)
}
func (tr transformer) String() string {
return fmt.Sprintf("Transformer(%s, %s)", tr.name, function.NameOf(tr.fnc))
}
// Comparer returns an [Option] that determines whether two values are equal
// to each other.
//
// The comparer f must be a function "func(T, T) bool" and is implicitly
// filtered to input values assignable to T. If T is an interface, it is
// possible that f is called with two values of different concrete types that
// both implement T.
//
// The equality function must be:
// - Symmetric: equal(x, y) == equal(y, x)
// - Deterministic: equal(x, y) == equal(x, y)
// - Pure: equal(x, y) does not modify x or y
func Comparer(f interface{}) Option {
v := reflect.ValueOf(f)
if !function.IsType(v.Type(), function.Equal) || v.IsNil() {
panic(fmt.Sprintf("invalid comparer function: %T", f))
}
cm := &comparer{fnc: v}
if ti := v.Type().In(0); ti.Kind() != reflect.Interface || ti.NumMethod() > 0 {
cm.typ = ti
}
return cm
}
type comparer struct {
core
typ reflect.Type // T
fnc reflect.Value // func(T, T) bool
}
func (cm *comparer) isFiltered() bool { return cm.typ != nil }
func (cm *comparer) filter(_ *state, t reflect.Type, _, _ reflect.Value) applicableOption {
if cm.typ == nil || t.AssignableTo(cm.typ) {
return cm
}
return nil
}
func (cm *comparer) apply(s *state, vx, vy reflect.Value) {
eq := s.callTTBFunc(cm.fnc, vx, vy)
s.report(eq, reportByFunc)
}
func (cm comparer) String() string {
return fmt.Sprintf("Comparer(%s)", function.NameOf(cm.fnc))
}
// Exporter returns an [Option] that specifies whether [Equal] is allowed to
// introspect into the unexported fields of certain struct types.
//
// Users of this option must understand that comparing on unexported fields
// from external packages is not safe since changes in the internal
// implementation of some external package may cause the result of [Equal]
// to unexpectedly change. However, it may be valid to use this option on types
// defined in an internal package where the semantic meaning of an unexported
// field is in the control of the user.
//
// In many cases, a custom [Comparer] should be used instead that defines
// equality as a function of the public API of a type rather than the underlying
// unexported implementation.
//
// For example, the [reflect.Type] documentation defines equality to be determined
// by the == operator on the interface (essentially performing a shallow pointer
// comparison) and most attempts to compare *[regexp.Regexp] types are interested
// in only checking that the regular expression strings are equal.
// Both of these are accomplished using [Comparer] options:
//
// Comparer(func(x, y reflect.Type) bool { return x == y })
// Comparer(func(x, y *regexp.Regexp) bool { return x.String() == y.String() })
//
// In other cases, the [github.com/google/go-cmp/cmp/cmpopts.IgnoreUnexported]
// option can be used to ignore all unexported fields on specified struct types.
func Exporter(f func(reflect.Type) bool) Option {
return exporter(f)
}
type exporter func(reflect.Type) bool
func (exporter) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
panic("not implemented")
}
// AllowUnexported returns an [Option] that allows [Equal] to forcibly introspect
// unexported fields of the specified struct types.
//
// See [Exporter] for the proper use of this option.
func AllowUnexported(types ...interface{}) Option {
m := make(map[reflect.Type]bool)
for _, typ := range types {
t := reflect.TypeOf(typ)
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("invalid struct type: %T", typ))
}
m[t] = true
}
return exporter(func(t reflect.Type) bool { return m[t] })
}
// Result represents the comparison result for a single node and
// is provided by cmp when calling Report (see [Reporter]).
type Result struct {
_ [0]func() // Make Result incomparable
flags resultFlags
}
// Equal reports whether the node was determined to be equal or not.
// As a special case, ignored nodes are considered equal.
func (r Result) Equal() bool {
return r.flags&(reportEqual|reportByIgnore) != 0
}
// ByIgnore reports whether the node is equal because it was ignored.
// This never reports true if [Result.Equal] reports false.
func (r Result) ByIgnore() bool {
return r.flags&reportByIgnore != 0
}
// ByMethod reports whether the Equal method determined equality.
func (r Result) ByMethod() bool {
return r.flags&reportByMethod != 0
}
// ByFunc reports whether a [Comparer] function determined equality.
func (r Result) ByFunc() bool {
return r.flags&reportByFunc != 0
}
// ByCycle reports whether a reference cycle was detected.
func (r Result) ByCycle() bool {
return r.flags&reportByCycle != 0
}
type resultFlags uint
const (
_ resultFlags = (1 << iota) / 2
reportEqual
reportUnequal
reportByIgnore
reportByMethod
reportByFunc
reportByCycle
)
// Reporter is an [Option] that can be passed to [Equal]. When [Equal] traverses
// the value trees, it calls PushStep as it descends into each node in the
// tree and PopStep as it ascend out of the node. The leaves of the tree are
// either compared (determined to be equal or not equal) or ignored and reported
// as such by calling the Report method.
func Reporter(r interface {
// PushStep is called when a tree-traversal operation is performed.
// The PathStep itself is only valid until the step is popped.
// The PathStep.Values are valid for the duration of the entire traversal
// and must not be mutated.
//
// Equal always calls PushStep at the start to provide an operation-less
// PathStep used to report the root values.
//
// Within a slice, the exact set of inserted, removed, or modified elements
// is unspecified and may change in future implementations.
// The entries of a map are iterated through in an unspecified order.
PushStep(PathStep)
// Report is called exactly once on leaf nodes to report whether the
// comparison identified the node as equal, unequal, or ignored.
// A leaf node is one that is immediately preceded by and followed by
// a pair of PushStep and PopStep calls.
Report(Result)
// PopStep ascends back up the value tree.
// There is always a matching pop call for every push call.
PopStep()
}) Option {
return reporter{r}
}
type reporter struct{ reporterIface }
type reporterIface interface {
PushStep(PathStep)
Report(Result)
PopStep()
}
func (reporter) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
panic("not implemented")
}
// normalizeOption normalizes the input options such that all Options groups
// are flattened and groups with a single element are reduced to that element.
// Only coreOptions and Options containing coreOptions are allowed.
func normalizeOption(src Option) Option {
switch opts := flattenOptions(nil, Options{src}); len(opts) {
case 0:
return nil
case 1:
return opts[0]
default:
return opts
}
}
// flattenOptions copies all options in src to dst as a flat list.
// Only coreOptions and Options containing coreOptions are allowed.
func flattenOptions(dst, src Options) Options {
for _, opt := range src {
switch opt := opt.(type) {
case nil:
continue
case Options:
dst = flattenOptions(dst, opt)
case coreOption:
dst = append(dst, opt)
default:
panic(fmt.Sprintf("invalid option type: %T", opt))
}
}
return dst
}

390
tools/tsgo/vendor/github.com/google/go-cmp/cmp/path.go generated vendored Normal file
View File

@@ -0,0 +1,390 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"fmt"
"reflect"
"strings"
"unicode"
"unicode/utf8"
"github.com/google/go-cmp/cmp/internal/value"
)
// Path is a list of [PathStep] describing the sequence of operations to get
// from some root type to the current position in the value tree.
// The first Path element is always an operation-less [PathStep] that exists
// simply to identify the initial type.
//
// When traversing structs with embedded structs, the embedded struct will
// always be accessed as a field before traversing the fields of the
// embedded struct themselves. That is, an exported field from the
// embedded struct will never be accessed directly from the parent struct.
type Path []PathStep
// PathStep is a union-type for specific operations to traverse
// a value's tree structure. Users of this package never need to implement
// these types as values of this type will be returned by this package.
//
// Implementations of this interface:
// - [StructField]
// - [SliceIndex]
// - [MapIndex]
// - [Indirect]
// - [TypeAssertion]
// - [Transform]
type PathStep interface {
String() string
// Type is the resulting type after performing the path step.
Type() reflect.Type
// Values is the resulting values after performing the path step.
// The type of each valid value is guaranteed to be identical to Type.
//
// In some cases, one or both may be invalid or have restrictions:
// - For StructField, both are not interface-able if the current field
// is unexported and the struct type is not explicitly permitted by
// an Exporter to traverse unexported fields.
// - For SliceIndex, one may be invalid if an element is missing from
// either the x or y slice.
// - For MapIndex, one may be invalid if an entry is missing from
// either the x or y map.
//
// The provided values must not be mutated.
Values() (vx, vy reflect.Value)
}
var (
_ PathStep = StructField{}
_ PathStep = SliceIndex{}
_ PathStep = MapIndex{}
_ PathStep = Indirect{}
_ PathStep = TypeAssertion{}
_ PathStep = Transform{}
)
func (pa *Path) push(s PathStep) {
*pa = append(*pa, s)
}
func (pa *Path) pop() {
*pa = (*pa)[:len(*pa)-1]
}
// Last returns the last [PathStep] in the Path.
// If the path is empty, this returns a non-nil [PathStep]
// that reports a nil [PathStep.Type].
func (pa Path) Last() PathStep {
return pa.Index(-1)
}
// Index returns the ith step in the Path and supports negative indexing.
// A negative index starts counting from the tail of the Path such that -1
// refers to the last step, -2 refers to the second-to-last step, and so on.
// If index is invalid, this returns a non-nil [PathStep]
// that reports a nil [PathStep.Type].
func (pa Path) Index(i int) PathStep {
if i < 0 {
i = len(pa) + i
}
if i < 0 || i >= len(pa) {
return pathStep{}
}
return pa[i]
}
// String returns the simplified path to a node.
// The simplified path only contains struct field accesses.
//
// For example:
//
// MyMap.MySlices.MyField
func (pa Path) String() string {
var ss []string
for _, s := range pa {
if _, ok := s.(StructField); ok {
ss = append(ss, s.String())
}
}
return strings.TrimPrefix(strings.Join(ss, ""), ".")
}
// GoString returns the path to a specific node using Go syntax.
//
// For example:
//
// (*root.MyMap["key"].(*mypkg.MyStruct).MySlices)[2][3].MyField
func (pa Path) GoString() string {
var ssPre, ssPost []string
var numIndirect int
for i, s := range pa {
var nextStep PathStep
if i+1 < len(pa) {
nextStep = pa[i+1]
}
switch s := s.(type) {
case Indirect:
numIndirect++
pPre, pPost := "(", ")"
switch nextStep.(type) {
case Indirect:
continue // Next step is indirection, so let them batch up
case StructField:
numIndirect-- // Automatic indirection on struct fields
case nil:
pPre, pPost = "", "" // Last step; no need for parenthesis
}
if numIndirect > 0 {
ssPre = append(ssPre, pPre+strings.Repeat("*", numIndirect))
ssPost = append(ssPost, pPost)
}
numIndirect = 0
continue
case Transform:
ssPre = append(ssPre, s.trans.name+"(")
ssPost = append(ssPost, ")")
continue
}
ssPost = append(ssPost, s.String())
}
for i, j := 0, len(ssPre)-1; i < j; i, j = i+1, j-1 {
ssPre[i], ssPre[j] = ssPre[j], ssPre[i]
}
return strings.Join(ssPre, "") + strings.Join(ssPost, "")
}
type pathStep struct {
typ reflect.Type
vx, vy reflect.Value
}
func (ps pathStep) Type() reflect.Type { return ps.typ }
func (ps pathStep) Values() (vx, vy reflect.Value) { return ps.vx, ps.vy }
func (ps pathStep) String() string {
if ps.typ == nil {
return "<nil>"
}
s := value.TypeString(ps.typ, false)
if s == "" || strings.ContainsAny(s, "{}\n") {
return "root" // Type too simple or complex to print
}
return fmt.Sprintf("{%s}", s)
}
// StructField is a [PathStep] that represents a struct field access
// on a field called [StructField.Name].
type StructField struct{ *structField }
type structField struct {
pathStep
name string
idx int
// These fields are used for forcibly accessing an unexported field.
// pvx, pvy, and field are only valid if unexported is true.
unexported bool
mayForce bool // Forcibly allow visibility
paddr bool // Was parent addressable?
pvx, pvy reflect.Value // Parent values (always addressable)
field reflect.StructField // Field information
}
func (sf StructField) Type() reflect.Type { return sf.typ }
func (sf StructField) Values() (vx, vy reflect.Value) {
if !sf.unexported {
return sf.vx, sf.vy // CanInterface reports true
}
// Forcibly obtain read-write access to an unexported struct field.
if sf.mayForce {
vx = retrieveUnexportedField(sf.pvx, sf.field, sf.paddr)
vy = retrieveUnexportedField(sf.pvy, sf.field, sf.paddr)
return vx, vy // CanInterface reports true
}
return sf.vx, sf.vy // CanInterface reports false
}
func (sf StructField) String() string { return fmt.Sprintf(".%s", sf.name) }
// Name is the field name.
func (sf StructField) Name() string { return sf.name }
// Index is the index of the field in the parent struct type.
// See [reflect.Type.Field].
func (sf StructField) Index() int { return sf.idx }
// SliceIndex is a [PathStep] that represents an index operation on
// a slice or array at some index [SliceIndex.Key].
type SliceIndex struct{ *sliceIndex }
type sliceIndex struct {
pathStep
xkey, ykey int
isSlice bool // False for reflect.Array
}
func (si SliceIndex) Type() reflect.Type { return si.typ }
func (si SliceIndex) Values() (vx, vy reflect.Value) { return si.vx, si.vy }
func (si SliceIndex) String() string {
switch {
case si.xkey == si.ykey:
return fmt.Sprintf("[%d]", si.xkey)
case si.ykey == -1:
// [5->?] means "I don't know where X[5] went"
return fmt.Sprintf("[%d->?]", si.xkey)
case si.xkey == -1:
// [?->3] means "I don't know where Y[3] came from"
return fmt.Sprintf("[?->%d]", si.ykey)
default:
// [5->3] means "X[5] moved to Y[3]"
return fmt.Sprintf("[%d->%d]", si.xkey, si.ykey)
}
}
// Key is the index key; it may return -1 if in a split state
func (si SliceIndex) Key() int {
if si.xkey != si.ykey {
return -1
}
return si.xkey
}
// SplitKeys are the indexes for indexing into slices in the
// x and y values, respectively. These indexes may differ due to the
// insertion or removal of an element in one of the slices, causing
// all of the indexes to be shifted. If an index is -1, then that
// indicates that the element does not exist in the associated slice.
//
// [SliceIndex.Key] is guaranteed to return -1 if and only if the indexes
// returned by SplitKeys are not the same. SplitKeys will never return -1 for
// both indexes.
func (si SliceIndex) SplitKeys() (ix, iy int) { return si.xkey, si.ykey }
// MapIndex is a [PathStep] that represents an index operation on a map at some index Key.
type MapIndex struct{ *mapIndex }
type mapIndex struct {
pathStep
key reflect.Value
}
func (mi MapIndex) Type() reflect.Type { return mi.typ }
func (mi MapIndex) Values() (vx, vy reflect.Value) { return mi.vx, mi.vy }
func (mi MapIndex) String() string { return fmt.Sprintf("[%#v]", mi.key) }
// Key is the value of the map key.
func (mi MapIndex) Key() reflect.Value { return mi.key }
// Indirect is a [PathStep] that represents pointer indirection on the parent type.
type Indirect struct{ *indirect }
type indirect struct {
pathStep
}
func (in Indirect) Type() reflect.Type { return in.typ }
func (in Indirect) Values() (vx, vy reflect.Value) { return in.vx, in.vy }
func (in Indirect) String() string { return "*" }
// TypeAssertion is a [PathStep] that represents a type assertion on an interface.
type TypeAssertion struct{ *typeAssertion }
type typeAssertion struct {
pathStep
}
func (ta TypeAssertion) Type() reflect.Type { return ta.typ }
func (ta TypeAssertion) Values() (vx, vy reflect.Value) { return ta.vx, ta.vy }
func (ta TypeAssertion) String() string { return fmt.Sprintf(".(%v)", value.TypeString(ta.typ, false)) }
// Transform is a [PathStep] that represents a transformation
// from the parent type to the current type.
type Transform struct{ *transform }
type transform struct {
pathStep
trans *transformer
}
func (tf Transform) Type() reflect.Type { return tf.typ }
func (tf Transform) Values() (vx, vy reflect.Value) { return tf.vx, tf.vy }
func (tf Transform) String() string { return fmt.Sprintf("%s()", tf.trans.name) }
// Name is the name of the [Transformer].
func (tf Transform) Name() string { return tf.trans.name }
// Func is the function pointer to the transformer function.
func (tf Transform) Func() reflect.Value { return tf.trans.fnc }
// Option returns the originally constructed [Transformer] option.
// The == operator can be used to detect the exact option used.
func (tf Transform) Option() Option { return tf.trans }
// pointerPath represents a dual-stack of pointers encountered when
// recursively traversing the x and y values. This data structure supports
// detection of cycles and determining whether the cycles are equal.
// In Go, cycles can occur via pointers, slices, and maps.
//
// The pointerPath uses a map to represent a stack; where descension into a
// pointer pushes the address onto the stack, and ascension from a pointer
// pops the address from the stack. Thus, when traversing into a pointer from
// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
// by checking whether the pointer has already been visited. The cycle detection
// uses a separate stack for the x and y values.
//
// If a cycle is detected we need to determine whether the two pointers
// should be considered equal. The definition of equality chosen by Equal
// requires two graphs to have the same structure. To determine this, both the
// x and y values must have a cycle where the previous pointers were also
// encountered together as a pair.
//
// Semantically, this is equivalent to augmenting Indirect, SliceIndex, and
// MapIndex with pointer information for the x and y values.
// Suppose px and py are two pointers to compare, we then search the
// Path for whether px was ever encountered in the Path history of x, and
// similarly so with py. If either side has a cycle, the comparison is only
// equal if both px and py have a cycle resulting from the same PathStep.
//
// Using a map as a stack is more performant as we can perform cycle detection
// in O(1) instead of O(N) where N is len(Path).
type pointerPath struct {
// mx is keyed by x pointers, where the value is the associated y pointer.
mx map[value.Pointer]value.Pointer
// my is keyed by y pointers, where the value is the associated x pointer.
my map[value.Pointer]value.Pointer
}
func (p *pointerPath) Init() {
p.mx = make(map[value.Pointer]value.Pointer)
p.my = make(map[value.Pointer]value.Pointer)
}
// Push indicates intent to descend into pointers vx and vy where
// visited reports whether either has been seen before. If visited before,
// equal reports whether both pointers were encountered together.
// Pop must be called if and only if the pointers were never visited.
//
// The pointers vx and vy must be a reflect.Ptr, reflect.Slice, or reflect.Map
// and be non-nil.
func (p pointerPath) Push(vx, vy reflect.Value) (equal, visited bool) {
px := value.PointerOf(vx)
py := value.PointerOf(vy)
_, ok1 := p.mx[px]
_, ok2 := p.my[py]
if ok1 || ok2 {
equal = p.mx[px] == py && p.my[py] == px // Pointers paired together
return equal, true
}
p.mx[px] = py
p.my[py] = px
return false, false
}
// Pop ascends from pointers vx and vy.
func (p pointerPath) Pop(vx, vy reflect.Value) {
delete(p.mx, value.PointerOf(vx))
delete(p.my, value.PointerOf(vy))
}
// isExported reports whether the identifier is exported.
func isExported(id string) bool {
r, _ := utf8.DecodeRuneInString(id)
return unicode.IsUpper(r)
}

View File

@@ -0,0 +1,54 @@
// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
// defaultReporter implements the reporter interface.
//
// As Equal serially calls the PushStep, Report, and PopStep methods, the
// defaultReporter constructs a tree-based representation of the compared value
// and the result of each comparison (see valueNode).
//
// When the String method is called, the FormatDiff method transforms the
// valueNode tree into a textNode tree, which is a tree-based representation
// of the textual output (see textNode).
//
// Lastly, the textNode.String method produces the final report as a string.
type defaultReporter struct {
root *valueNode
curr *valueNode
}
func (r *defaultReporter) PushStep(ps PathStep) {
r.curr = r.curr.PushStep(ps)
if r.root == nil {
r.root = r.curr
}
}
func (r *defaultReporter) Report(rs Result) {
r.curr.Report(rs)
}
func (r *defaultReporter) PopStep() {
r.curr = r.curr.PopStep()
}
// String provides a full report of the differences detected as a structured
// literal in pseudo-Go syntax. String may only be called after the entire tree
// has been traversed.
func (r *defaultReporter) String() string {
assert(r.root != nil && r.curr == nil)
if r.root.NumDiff == 0 {
return ""
}
ptrs := new(pointerReferences)
text := formatOptions{}.FormatDiff(r.root, ptrs)
resolveReferences(text)
return text.String()
}
func assert(ok bool) {
if !ok {
panic("assertion failure")
}
}

View File

@@ -0,0 +1,433 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"fmt"
"reflect"
)
// numContextRecords is the number of surrounding equal records to print.
const numContextRecords = 2
type diffMode byte
const (
diffUnknown diffMode = 0
diffIdentical diffMode = ' '
diffRemoved diffMode = '-'
diffInserted diffMode = '+'
)
type typeMode int
const (
// emitType always prints the type.
emitType typeMode = iota
// elideType never prints the type.
elideType
// autoType prints the type only for composite kinds
// (i.e., structs, slices, arrays, and maps).
autoType
)
type formatOptions struct {
// DiffMode controls the output mode of FormatDiff.
//
// If diffUnknown, then produce a diff of the x and y values.
// If diffIdentical, then emit values as if they were equal.
// If diffRemoved, then only emit x values (ignoring y values).
// If diffInserted, then only emit y values (ignoring x values).
DiffMode diffMode
// TypeMode controls whether to print the type for the current node.
//
// As a general rule of thumb, we always print the type of the next node
// after an interface, and always elide the type of the next node after
// a slice or map node.
TypeMode typeMode
// formatValueOptions are options specific to printing reflect.Values.
formatValueOptions
}
func (opts formatOptions) WithDiffMode(d diffMode) formatOptions {
opts.DiffMode = d
return opts
}
func (opts formatOptions) WithTypeMode(t typeMode) formatOptions {
opts.TypeMode = t
return opts
}
func (opts formatOptions) WithVerbosity(level int) formatOptions {
opts.VerbosityLevel = level
opts.LimitVerbosity = true
return opts
}
func (opts formatOptions) verbosity() uint {
switch {
case opts.VerbosityLevel < 0:
return 0
case opts.VerbosityLevel > 16:
return 16 // some reasonable maximum to avoid shift overflow
default:
return uint(opts.VerbosityLevel)
}
}
const maxVerbosityPreset = 6
// verbosityPreset modifies the verbosity settings given an index
// between 0 and maxVerbosityPreset, inclusive.
func verbosityPreset(opts formatOptions, i int) formatOptions {
opts.VerbosityLevel = int(opts.verbosity()) + 2*i
if i > 0 {
opts.AvoidStringer = true
}
if i >= maxVerbosityPreset {
opts.PrintAddresses = true
opts.QualifiedNames = true
}
return opts
}
// FormatDiff converts a valueNode tree into a textNode tree, where the later
// is a textual representation of the differences detected in the former.
func (opts formatOptions) FormatDiff(v *valueNode, ptrs *pointerReferences) (out textNode) {
if opts.DiffMode == diffIdentical {
opts = opts.WithVerbosity(1)
} else if opts.verbosity() < 3 {
opts = opts.WithVerbosity(3)
}
// Check whether we have specialized formatting for this node.
// This is not necessary, but helpful for producing more readable outputs.
if opts.CanFormatDiffSlice(v) {
return opts.FormatDiffSlice(v)
}
var parentKind reflect.Kind
if v.parent != nil && v.parent.TransformerName == "" {
parentKind = v.parent.Type.Kind()
}
// For leaf nodes, format the value based on the reflect.Values alone.
// As a special case, treat equal []byte as a leaf nodes.
isBytes := v.Type.Kind() == reflect.Slice && v.Type.Elem() == byteType
isEqualBytes := isBytes && v.NumDiff+v.NumIgnored+v.NumTransformed == 0
if v.MaxDepth == 0 || isEqualBytes {
switch opts.DiffMode {
case diffUnknown, diffIdentical:
// Format Equal.
if v.NumDiff == 0 {
outx := opts.FormatValue(v.ValueX, parentKind, ptrs)
outy := opts.FormatValue(v.ValueY, parentKind, ptrs)
if v.NumIgnored > 0 && v.NumSame == 0 {
return textEllipsis
} else if outx.Len() < outy.Len() {
return outx
} else {
return outy
}
}
// Format unequal.
assert(opts.DiffMode == diffUnknown)
var list textList
outx := opts.WithTypeMode(elideType).FormatValue(v.ValueX, parentKind, ptrs)
outy := opts.WithTypeMode(elideType).FormatValue(v.ValueY, parentKind, ptrs)
for i := 0; i <= maxVerbosityPreset && outx != nil && outy != nil && outx.Equal(outy); i++ {
opts2 := verbosityPreset(opts, i).WithTypeMode(elideType)
outx = opts2.FormatValue(v.ValueX, parentKind, ptrs)
outy = opts2.FormatValue(v.ValueY, parentKind, ptrs)
}
if outx != nil {
list = append(list, textRecord{Diff: '-', Value: outx})
}
if outy != nil {
list = append(list, textRecord{Diff: '+', Value: outy})
}
return opts.WithTypeMode(emitType).FormatType(v.Type, list)
case diffRemoved:
return opts.FormatValue(v.ValueX, parentKind, ptrs)
case diffInserted:
return opts.FormatValue(v.ValueY, parentKind, ptrs)
default:
panic("invalid diff mode")
}
}
// Register slice element to support cycle detection.
if parentKind == reflect.Slice {
ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, true)
defer ptrs.Pop()
defer func() { out = wrapTrunkReferences(ptrRefs, out) }()
}
// Descend into the child value node.
if v.TransformerName != "" {
out := opts.WithTypeMode(emitType).FormatDiff(v.Value, ptrs)
out = &textWrap{Prefix: "Inverse(" + v.TransformerName + ", ", Value: out, Suffix: ")"}
return opts.FormatType(v.Type, out)
} else {
switch k := v.Type.Kind(); k {
case reflect.Struct, reflect.Array, reflect.Slice:
out = opts.formatDiffList(v.Records, k, ptrs)
out = opts.FormatType(v.Type, out)
case reflect.Map:
// Register map to support cycle detection.
ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, false)
defer ptrs.Pop()
out = opts.formatDiffList(v.Records, k, ptrs)
out = wrapTrunkReferences(ptrRefs, out)
out = opts.FormatType(v.Type, out)
case reflect.Ptr:
// Register pointer to support cycle detection.
ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, false)
defer ptrs.Pop()
out = opts.FormatDiff(v.Value, ptrs)
out = wrapTrunkReferences(ptrRefs, out)
out = &textWrap{Prefix: "&", Value: out}
case reflect.Interface:
out = opts.WithTypeMode(emitType).FormatDiff(v.Value, ptrs)
default:
panic(fmt.Sprintf("%v cannot have children", k))
}
return out
}
}
func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind, ptrs *pointerReferences) textNode {
// Derive record name based on the data structure kind.
var name string
var formatKey func(reflect.Value) string
switch k {
case reflect.Struct:
name = "field"
opts = opts.WithTypeMode(autoType)
formatKey = func(v reflect.Value) string { return v.String() }
case reflect.Slice, reflect.Array:
name = "element"
opts = opts.WithTypeMode(elideType)
formatKey = func(reflect.Value) string { return "" }
case reflect.Map:
name = "entry"
opts = opts.WithTypeMode(elideType)
formatKey = func(v reflect.Value) string { return formatMapKey(v, false, ptrs) }
}
maxLen := -1
if opts.LimitVerbosity {
if opts.DiffMode == diffIdentical {
maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
} else {
maxLen = (1 << opts.verbosity()) << 1 // 2, 4, 8, 16, 32, 64, etc...
}
opts.VerbosityLevel--
}
// Handle unification.
switch opts.DiffMode {
case diffIdentical, diffRemoved, diffInserted:
var list textList
var deferredEllipsis bool // Add final "..." to indicate records were dropped
for _, r := range recs {
if len(list) == maxLen {
deferredEllipsis = true
break
}
// Elide struct fields that are zero value.
if k == reflect.Struct {
var isZero bool
switch opts.DiffMode {
case diffIdentical:
isZero = r.Value.ValueX.IsZero() || r.Value.ValueY.IsZero()
case diffRemoved:
isZero = r.Value.ValueX.IsZero()
case diffInserted:
isZero = r.Value.ValueY.IsZero()
}
if isZero {
continue
}
}
// Elide ignored nodes.
if r.Value.NumIgnored > 0 && r.Value.NumSame+r.Value.NumDiff == 0 {
deferredEllipsis = !(k == reflect.Slice || k == reflect.Array)
if !deferredEllipsis {
list.AppendEllipsis(diffStats{})
}
continue
}
if out := opts.FormatDiff(r.Value, ptrs); out != nil {
list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
}
}
if deferredEllipsis {
list.AppendEllipsis(diffStats{})
}
return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
case diffUnknown:
default:
panic("invalid diff mode")
}
// Handle differencing.
var numDiffs int
var list textList
var keys []reflect.Value // invariant: len(list) == len(keys)
groups := coalesceAdjacentRecords(name, recs)
maxGroup := diffStats{Name: name}
for i, ds := range groups {
if maxLen >= 0 && numDiffs >= maxLen {
maxGroup = maxGroup.Append(ds)
continue
}
// Handle equal records.
if ds.NumDiff() == 0 {
// Compute the number of leading and trailing records to print.
var numLo, numHi int
numEqual := ds.NumIgnored + ds.NumIdentical
for numLo < numContextRecords && numLo+numHi < numEqual && i != 0 {
if r := recs[numLo].Value; r.NumIgnored > 0 && r.NumSame+r.NumDiff == 0 {
break
}
numLo++
}
for numHi < numContextRecords && numLo+numHi < numEqual && i != len(groups)-1 {
if r := recs[numEqual-numHi-1].Value; r.NumIgnored > 0 && r.NumSame+r.NumDiff == 0 {
break
}
numHi++
}
if numEqual-(numLo+numHi) == 1 && ds.NumIgnored == 0 {
numHi++ // Avoid pointless coalescing of a single equal record
}
// Format the equal values.
for _, r := range recs[:numLo] {
out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value, ptrs)
list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
keys = append(keys, r.Key)
}
if numEqual > numLo+numHi {
ds.NumIdentical -= numLo + numHi
list.AppendEllipsis(ds)
for len(keys) < len(list) {
keys = append(keys, reflect.Value{})
}
}
for _, r := range recs[numEqual-numHi : numEqual] {
out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value, ptrs)
list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
keys = append(keys, r.Key)
}
recs = recs[numEqual:]
continue
}
// Handle unequal records.
for _, r := range recs[:ds.NumDiff()] {
switch {
case opts.CanFormatDiffSlice(r.Value):
out := opts.FormatDiffSlice(r.Value)
list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
keys = append(keys, r.Key)
case r.Value.NumChildren == r.Value.MaxDepth:
outx := opts.WithDiffMode(diffRemoved).FormatDiff(r.Value, ptrs)
outy := opts.WithDiffMode(diffInserted).FormatDiff(r.Value, ptrs)
for i := 0; i <= maxVerbosityPreset && outx != nil && outy != nil && outx.Equal(outy); i++ {
opts2 := verbosityPreset(opts, i)
outx = opts2.WithDiffMode(diffRemoved).FormatDiff(r.Value, ptrs)
outy = opts2.WithDiffMode(diffInserted).FormatDiff(r.Value, ptrs)
}
if outx != nil {
list = append(list, textRecord{Diff: diffRemoved, Key: formatKey(r.Key), Value: outx})
keys = append(keys, r.Key)
}
if outy != nil {
list = append(list, textRecord{Diff: diffInserted, Key: formatKey(r.Key), Value: outy})
keys = append(keys, r.Key)
}
default:
out := opts.FormatDiff(r.Value, ptrs)
list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
keys = append(keys, r.Key)
}
}
recs = recs[ds.NumDiff():]
numDiffs += ds.NumDiff()
}
if maxGroup.IsZero() {
assert(len(recs) == 0)
} else {
list.AppendEllipsis(maxGroup)
for len(keys) < len(list) {
keys = append(keys, reflect.Value{})
}
}
assert(len(list) == len(keys))
// For maps, the default formatting logic uses fmt.Stringer which may
// produce ambiguous output. Avoid calling String to disambiguate.
if k == reflect.Map {
var ambiguous bool
seenKeys := map[string]reflect.Value{}
for i, currKey := range keys {
if currKey.IsValid() {
strKey := list[i].Key
prevKey, seen := seenKeys[strKey]
if seen && prevKey.CanInterface() && currKey.CanInterface() {
ambiguous = prevKey.Interface() != currKey.Interface()
if ambiguous {
break
}
}
seenKeys[strKey] = currKey
}
}
if ambiguous {
for i, k := range keys {
if k.IsValid() {
list[i].Key = formatMapKey(k, true, ptrs)
}
}
}
}
return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
}
// coalesceAdjacentRecords coalesces the list of records into groups of
// adjacent equal, or unequal counts.
func coalesceAdjacentRecords(name string, recs []reportRecord) (groups []diffStats) {
var prevCase int // Arbitrary index into which case last occurred
lastStats := func(i int) *diffStats {
if prevCase != i {
groups = append(groups, diffStats{Name: name})
prevCase = i
}
return &groups[len(groups)-1]
}
for _, r := range recs {
switch rv := r.Value; {
case rv.NumIgnored > 0 && rv.NumSame+rv.NumDiff == 0:
lastStats(1).NumIgnored++
case rv.NumDiff == 0:
lastStats(1).NumIdentical++
case rv.NumDiff > 0 && !rv.ValueY.IsValid():
lastStats(2).NumRemoved++
case rv.NumDiff > 0 && !rv.ValueX.IsValid():
lastStats(2).NumInserted++
default:
lastStats(2).NumModified++
}
}
return groups
}

View File

@@ -0,0 +1,264 @@
// Copyright 2020, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"fmt"
"reflect"
"strings"
"github.com/google/go-cmp/cmp/internal/flags"
"github.com/google/go-cmp/cmp/internal/value"
)
const (
pointerDelimPrefix = "⟪"
pointerDelimSuffix = "⟫"
)
// formatPointer prints the address of the pointer.
func formatPointer(p value.Pointer, withDelims bool) string {
v := p.Uintptr()
if flags.Deterministic {
v = 0xdeadf00f // Only used for stable testing purposes
}
if withDelims {
return pointerDelimPrefix + formatHex(uint64(v)) + pointerDelimSuffix
}
return formatHex(uint64(v))
}
// pointerReferences is a stack of pointers visited so far.
type pointerReferences [][2]value.Pointer
func (ps *pointerReferences) PushPair(vx, vy reflect.Value, d diffMode, deref bool) (pp [2]value.Pointer) {
if deref && vx.IsValid() {
vx = vx.Addr()
}
if deref && vy.IsValid() {
vy = vy.Addr()
}
switch d {
case diffUnknown, diffIdentical:
pp = [2]value.Pointer{value.PointerOf(vx), value.PointerOf(vy)}
case diffRemoved:
pp = [2]value.Pointer{value.PointerOf(vx), value.Pointer{}}
case diffInserted:
pp = [2]value.Pointer{value.Pointer{}, value.PointerOf(vy)}
}
*ps = append(*ps, pp)
return pp
}
func (ps *pointerReferences) Push(v reflect.Value) (p value.Pointer, seen bool) {
p = value.PointerOf(v)
for _, pp := range *ps {
if p == pp[0] || p == pp[1] {
return p, true
}
}
*ps = append(*ps, [2]value.Pointer{p, p})
return p, false
}
func (ps *pointerReferences) Pop() {
*ps = (*ps)[:len(*ps)-1]
}
// trunkReferences is metadata for a textNode indicating that the sub-tree
// represents the value for either pointer in a pair of references.
type trunkReferences struct{ pp [2]value.Pointer }
// trunkReference is metadata for a textNode indicating that the sub-tree
// represents the value for the given pointer reference.
type trunkReference struct{ p value.Pointer }
// leafReference is metadata for a textNode indicating that the value is
// truncated as it refers to another part of the tree (i.e., a trunk).
type leafReference struct{ p value.Pointer }
func wrapTrunkReferences(pp [2]value.Pointer, s textNode) textNode {
switch {
case pp[0].IsNil():
return &textWrap{Value: s, Metadata: trunkReference{pp[1]}}
case pp[1].IsNil():
return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
case pp[0] == pp[1]:
return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
default:
return &textWrap{Value: s, Metadata: trunkReferences{pp}}
}
}
func wrapTrunkReference(p value.Pointer, printAddress bool, s textNode) textNode {
var prefix string
if printAddress {
prefix = formatPointer(p, true)
}
return &textWrap{Prefix: prefix, Value: s, Metadata: trunkReference{p}}
}
func makeLeafReference(p value.Pointer, printAddress bool) textNode {
out := &textWrap{Prefix: "(", Value: textEllipsis, Suffix: ")"}
var prefix string
if printAddress {
prefix = formatPointer(p, true)
}
return &textWrap{Prefix: prefix, Value: out, Metadata: leafReference{p}}
}
// resolveReferences walks the textNode tree searching for any leaf reference
// metadata and resolves each against the corresponding trunk references.
// Since pointer addresses in memory are not particularly readable to the user,
// it replaces each pointer value with an arbitrary and unique reference ID.
func resolveReferences(s textNode) {
var walkNodes func(textNode, func(textNode))
walkNodes = func(s textNode, f func(textNode)) {
f(s)
switch s := s.(type) {
case *textWrap:
walkNodes(s.Value, f)
case textList:
for _, r := range s {
walkNodes(r.Value, f)
}
}
}
// Collect all trunks and leaves with reference metadata.
var trunks, leaves []*textWrap
walkNodes(s, func(s textNode) {
if s, ok := s.(*textWrap); ok {
switch s.Metadata.(type) {
case leafReference:
leaves = append(leaves, s)
case trunkReference, trunkReferences:
trunks = append(trunks, s)
}
}
})
// No leaf references to resolve.
if len(leaves) == 0 {
return
}
// Collect the set of all leaf references to resolve.
leafPtrs := make(map[value.Pointer]bool)
for _, leaf := range leaves {
leafPtrs[leaf.Metadata.(leafReference).p] = true
}
// Collect the set of trunk pointers that are always paired together.
// This allows us to assign a single ID to both pointers for brevity.
// If a pointer in a pair ever occurs by itself or as a different pair,
// then the pair is broken.
pairedTrunkPtrs := make(map[value.Pointer]value.Pointer)
unpair := func(p value.Pointer) {
if !pairedTrunkPtrs[p].IsNil() {
pairedTrunkPtrs[pairedTrunkPtrs[p]] = value.Pointer{} // invalidate other half
}
pairedTrunkPtrs[p] = value.Pointer{} // invalidate this half
}
for _, trunk := range trunks {
switch p := trunk.Metadata.(type) {
case trunkReference:
unpair(p.p) // standalone pointer cannot be part of a pair
case trunkReferences:
p0, ok0 := pairedTrunkPtrs[p.pp[0]]
p1, ok1 := pairedTrunkPtrs[p.pp[1]]
switch {
case !ok0 && !ok1:
// Register the newly seen pair.
pairedTrunkPtrs[p.pp[0]] = p.pp[1]
pairedTrunkPtrs[p.pp[1]] = p.pp[0]
case ok0 && ok1 && p0 == p.pp[1] && p1 == p.pp[0]:
// Exact pair already seen; do nothing.
default:
// Pair conflicts with some other pair; break all pairs.
unpair(p.pp[0])
unpair(p.pp[1])
}
}
}
// Correlate each pointer referenced by leaves to a unique identifier,
// and print the IDs for each trunk that matches those pointers.
var nextID uint
ptrIDs := make(map[value.Pointer]uint)
newID := func() uint {
id := nextID
nextID++
return id
}
for _, trunk := range trunks {
switch p := trunk.Metadata.(type) {
case trunkReference:
if print := leafPtrs[p.p]; print {
id, ok := ptrIDs[p.p]
if !ok {
id = newID()
ptrIDs[p.p] = id
}
trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
}
case trunkReferences:
print0 := leafPtrs[p.pp[0]]
print1 := leafPtrs[p.pp[1]]
if print0 || print1 {
id0, ok0 := ptrIDs[p.pp[0]]
id1, ok1 := ptrIDs[p.pp[1]]
isPair := pairedTrunkPtrs[p.pp[0]] == p.pp[1] && pairedTrunkPtrs[p.pp[1]] == p.pp[0]
if isPair {
var id uint
assert(ok0 == ok1) // must be seen together or not at all
if ok0 {
assert(id0 == id1) // must have the same ID
id = id0
} else {
id = newID()
ptrIDs[p.pp[0]] = id
ptrIDs[p.pp[1]] = id
}
trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
} else {
if print0 && !ok0 {
id0 = newID()
ptrIDs[p.pp[0]] = id0
}
if print1 && !ok1 {
id1 = newID()
ptrIDs[p.pp[1]] = id1
}
switch {
case print0 && print1:
trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0)+","+formatReference(id1))
case print0:
trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0))
case print1:
trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id1))
}
}
}
}
}
// Update all leaf references with the unique identifier.
for _, leaf := range leaves {
if id, ok := ptrIDs[leaf.Metadata.(leafReference).p]; ok {
leaf.Prefix = updateReferencePrefix(leaf.Prefix, formatReference(id))
}
}
}
func formatReference(id uint) string {
return fmt.Sprintf("ref#%d", id)
}
func updateReferencePrefix(prefix, ref string) string {
if prefix == "" {
return pointerDelimPrefix + ref + pointerDelimSuffix
}
suffix := strings.TrimPrefix(prefix, pointerDelimPrefix)
return pointerDelimPrefix + ref + ": " + suffix
}

View File

@@ -0,0 +1,414 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"bytes"
"fmt"
"reflect"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/google/go-cmp/cmp/internal/value"
)
var (
anyType = reflect.TypeOf((*interface{})(nil)).Elem()
stringType = reflect.TypeOf((*string)(nil)).Elem()
bytesType = reflect.TypeOf((*[]byte)(nil)).Elem()
byteType = reflect.TypeOf((*byte)(nil)).Elem()
)
type formatValueOptions struct {
// AvoidStringer controls whether to avoid calling custom stringer
// methods like error.Error or fmt.Stringer.String.
AvoidStringer bool
// PrintAddresses controls whether to print the address of all pointers,
// slice elements, and maps.
PrintAddresses bool
// QualifiedNames controls whether FormatType uses the fully qualified name
// (including the full package path as opposed to just the package name).
QualifiedNames bool
// VerbosityLevel controls the amount of output to produce.
// A higher value produces more output. A value of zero or lower produces
// no output (represented using an ellipsis).
// If LimitVerbosity is false, then the level is treated as infinite.
VerbosityLevel int
// LimitVerbosity specifies that formatting should respect VerbosityLevel.
LimitVerbosity bool
}
// FormatType prints the type as if it were wrapping s.
// This may return s as-is depending on the current type and TypeMode mode.
func (opts formatOptions) FormatType(t reflect.Type, s textNode) textNode {
// Check whether to emit the type or not.
switch opts.TypeMode {
case autoType:
switch t.Kind() {
case reflect.Struct, reflect.Slice, reflect.Array, reflect.Map:
if s.Equal(textNil) {
return s
}
default:
return s
}
if opts.DiffMode == diffIdentical {
return s // elide type for identical nodes
}
case elideType:
return s
}
// Determine the type label, applying special handling for unnamed types.
typeName := value.TypeString(t, opts.QualifiedNames)
if t.Name() == "" {
// According to Go grammar, certain type literals contain symbols that
// do not strongly bind to the next lexicographical token (e.g., *T).
switch t.Kind() {
case reflect.Chan, reflect.Func, reflect.Ptr:
typeName = "(" + typeName + ")"
}
}
return &textWrap{Prefix: typeName, Value: wrapParens(s)}
}
// wrapParens wraps s with a set of parenthesis, but avoids it if the
// wrapped node itself is already surrounded by a pair of parenthesis or braces.
// It handles unwrapping one level of pointer-reference nodes.
func wrapParens(s textNode) textNode {
var refNode *textWrap
if s2, ok := s.(*textWrap); ok {
// Unwrap a single pointer reference node.
switch s2.Metadata.(type) {
case leafReference, trunkReference, trunkReferences:
refNode = s2
if s3, ok := refNode.Value.(*textWrap); ok {
s2 = s3
}
}
// Already has delimiters that make parenthesis unnecessary.
hasParens := strings.HasPrefix(s2.Prefix, "(") && strings.HasSuffix(s2.Suffix, ")")
hasBraces := strings.HasPrefix(s2.Prefix, "{") && strings.HasSuffix(s2.Suffix, "}")
if hasParens || hasBraces {
return s
}
}
if refNode != nil {
refNode.Value = &textWrap{Prefix: "(", Value: refNode.Value, Suffix: ")"}
return s
}
return &textWrap{Prefix: "(", Value: s, Suffix: ")"}
}
// FormatValue prints the reflect.Value, taking extra care to avoid descending
// into pointers already in ptrs. As pointers are visited, ptrs is also updated.
func (opts formatOptions) FormatValue(v reflect.Value, parentKind reflect.Kind, ptrs *pointerReferences) (out textNode) {
if !v.IsValid() {
return nil
}
t := v.Type()
// Check slice element for cycles.
if parentKind == reflect.Slice {
ptrRef, visited := ptrs.Push(v.Addr())
if visited {
return makeLeafReference(ptrRef, false)
}
defer ptrs.Pop()
defer func() { out = wrapTrunkReference(ptrRef, false, out) }()
}
// Check whether there is an Error or String method to call.
if !opts.AvoidStringer && v.CanInterface() {
// Avoid calling Error or String methods on nil receivers since many
// implementations crash when doing so.
if (t.Kind() != reflect.Ptr && t.Kind() != reflect.Interface) || !v.IsNil() {
var prefix, strVal string
func() {
// Swallow and ignore any panics from String or Error.
defer func() { recover() }()
switch v := v.Interface().(type) {
case error:
strVal = v.Error()
prefix = "e"
case fmt.Stringer:
strVal = v.String()
prefix = "s"
}
}()
if prefix != "" {
return opts.formatString(prefix, strVal)
}
}
}
// Check whether to explicitly wrap the result with the type.
var skipType bool
defer func() {
if !skipType {
out = opts.FormatType(t, out)
}
}()
switch t.Kind() {
case reflect.Bool:
return textLine(fmt.Sprint(v.Bool()))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return textLine(fmt.Sprint(v.Int()))
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return textLine(fmt.Sprint(v.Uint()))
case reflect.Uint8:
if parentKind == reflect.Slice || parentKind == reflect.Array {
return textLine(formatHex(v.Uint()))
}
return textLine(fmt.Sprint(v.Uint()))
case reflect.Uintptr:
return textLine(formatHex(v.Uint()))
case reflect.Float32, reflect.Float64:
return textLine(fmt.Sprint(v.Float()))
case reflect.Complex64, reflect.Complex128:
return textLine(fmt.Sprint(v.Complex()))
case reflect.String:
return opts.formatString("", v.String())
case reflect.UnsafePointer, reflect.Chan, reflect.Func:
return textLine(formatPointer(value.PointerOf(v), true))
case reflect.Struct:
var list textList
v := makeAddressable(v) // needed for retrieveUnexportedField
maxLen := v.NumField()
if opts.LimitVerbosity {
maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
opts.VerbosityLevel--
}
for i := 0; i < v.NumField(); i++ {
vv := v.Field(i)
if vv.IsZero() {
continue // Elide fields with zero values
}
if len(list) == maxLen {
list.AppendEllipsis(diffStats{})
break
}
sf := t.Field(i)
if !isExported(sf.Name) {
vv = retrieveUnexportedField(v, sf, true)
}
s := opts.WithTypeMode(autoType).FormatValue(vv, t.Kind(), ptrs)
list = append(list, textRecord{Key: sf.Name, Value: s})
}
return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
case reflect.Slice:
if v.IsNil() {
return textNil
}
// Check whether this is a []byte of text data.
if t.Elem() == byteType {
b := v.Bytes()
isPrintSpace := func(r rune) bool { return unicode.IsPrint(r) || unicode.IsSpace(r) }
if len(b) > 0 && utf8.Valid(b) && len(bytes.TrimFunc(b, isPrintSpace)) == 0 {
out = opts.formatString("", string(b))
skipType = true
return opts.FormatType(t, out)
}
}
fallthrough
case reflect.Array:
maxLen := v.Len()
if opts.LimitVerbosity {
maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
opts.VerbosityLevel--
}
var list textList
for i := 0; i < v.Len(); i++ {
if len(list) == maxLen {
list.AppendEllipsis(diffStats{})
break
}
s := opts.WithTypeMode(elideType).FormatValue(v.Index(i), t.Kind(), ptrs)
list = append(list, textRecord{Value: s})
}
out = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
if t.Kind() == reflect.Slice && opts.PrintAddresses {
header := fmt.Sprintf("ptr:%v, len:%d, cap:%d", formatPointer(value.PointerOf(v), false), v.Len(), v.Cap())
out = &textWrap{Prefix: pointerDelimPrefix + header + pointerDelimSuffix, Value: out}
}
return out
case reflect.Map:
if v.IsNil() {
return textNil
}
// Check pointer for cycles.
ptrRef, visited := ptrs.Push(v)
if visited {
return makeLeafReference(ptrRef, opts.PrintAddresses)
}
defer ptrs.Pop()
maxLen := v.Len()
if opts.LimitVerbosity {
maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
opts.VerbosityLevel--
}
var list textList
for _, k := range value.SortKeys(v.MapKeys()) {
if len(list) == maxLen {
list.AppendEllipsis(diffStats{})
break
}
sk := formatMapKey(k, false, ptrs)
sv := opts.WithTypeMode(elideType).FormatValue(v.MapIndex(k), t.Kind(), ptrs)
list = append(list, textRecord{Key: sk, Value: sv})
}
out = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
out = wrapTrunkReference(ptrRef, opts.PrintAddresses, out)
return out
case reflect.Ptr:
if v.IsNil() {
return textNil
}
// Check pointer for cycles.
ptrRef, visited := ptrs.Push(v)
if visited {
out = makeLeafReference(ptrRef, opts.PrintAddresses)
return &textWrap{Prefix: "&", Value: out}
}
defer ptrs.Pop()
// Skip the name only if this is an unnamed pointer type.
// Otherwise taking the address of a value does not reproduce
// the named pointer type.
if v.Type().Name() == "" {
skipType = true // Let the underlying value print the type instead
}
out = opts.FormatValue(v.Elem(), t.Kind(), ptrs)
out = wrapTrunkReference(ptrRef, opts.PrintAddresses, out)
out = &textWrap{Prefix: "&", Value: out}
return out
case reflect.Interface:
if v.IsNil() {
return textNil
}
// Interfaces accept different concrete types,
// so configure the underlying value to explicitly print the type.
return opts.WithTypeMode(emitType).FormatValue(v.Elem(), t.Kind(), ptrs)
default:
panic(fmt.Sprintf("%v kind not handled", v.Kind()))
}
}
func (opts formatOptions) formatString(prefix, s string) textNode {
maxLen := len(s)
maxLines := strings.Count(s, "\n") + 1
if opts.LimitVerbosity {
maxLen = (1 << opts.verbosity()) << 5 // 32, 64, 128, 256, etc...
maxLines = (1 << opts.verbosity()) << 2 // 4, 8, 16, 32, 64, etc...
}
// For multiline strings, use the triple-quote syntax,
// but only use it when printing removed or inserted nodes since
// we only want the extra verbosity for those cases.
lines := strings.Split(strings.TrimSuffix(s, "\n"), "\n")
isTripleQuoted := len(lines) >= 4 && (opts.DiffMode == '-' || opts.DiffMode == '+')
for i := 0; i < len(lines) && isTripleQuoted; i++ {
lines[i] = strings.TrimPrefix(strings.TrimSuffix(lines[i], "\r"), "\r") // trim leading/trailing carriage returns for legacy Windows endline support
isPrintable := func(r rune) bool {
return unicode.IsPrint(r) || r == '\t' // specially treat tab as printable
}
line := lines[i]
isTripleQuoted = !strings.HasPrefix(strings.TrimPrefix(line, prefix), `"""`) && !strings.HasPrefix(line, "...") && strings.TrimFunc(line, isPrintable) == "" && len(line) <= maxLen
}
if isTripleQuoted {
var list textList
list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(prefix + `"""`), ElideComma: true})
for i, line := range lines {
if numElided := len(lines) - i; i == maxLines-1 && numElided > 1 {
comment := commentString(fmt.Sprintf("%d elided lines", numElided))
list = append(list, textRecord{Diff: opts.DiffMode, Value: textEllipsis, ElideComma: true, Comment: comment})
break
}
list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(line), ElideComma: true})
}
list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(prefix + `"""`), ElideComma: true})
return &textWrap{Prefix: "(", Value: list, Suffix: ")"}
}
// Format the string as a single-line quoted string.
if len(s) > maxLen+len(textEllipsis) {
return textLine(prefix + formatString(s[:maxLen]) + string(textEllipsis))
}
return textLine(prefix + formatString(s))
}
// formatMapKey formats v as if it were a map key.
// The result is guaranteed to be a single line.
func formatMapKey(v reflect.Value, disambiguate bool, ptrs *pointerReferences) string {
var opts formatOptions
opts.DiffMode = diffIdentical
opts.TypeMode = elideType
opts.PrintAddresses = disambiguate
opts.AvoidStringer = disambiguate
opts.QualifiedNames = disambiguate
opts.VerbosityLevel = maxVerbosityPreset
opts.LimitVerbosity = true
s := opts.FormatValue(v, reflect.Map, ptrs).String()
return strings.TrimSpace(s)
}
// formatString prints s as a double-quoted or backtick-quoted string.
func formatString(s string) string {
// Use quoted string if it the same length as a raw string literal.
// Otherwise, attempt to use the raw string form.
qs := strconv.Quote(s)
if len(qs) == 1+len(s)+1 {
return qs
}
// Disallow newlines to ensure output is a single line.
// Only allow printable runes for readability purposes.
rawInvalid := func(r rune) bool {
return r == '`' || r == '\n' || !(unicode.IsPrint(r) || r == '\t')
}
if utf8.ValidString(s) && strings.IndexFunc(s, rawInvalid) < 0 {
return "`" + s + "`"
}
return qs
}
// formatHex prints u as a hexadecimal integer in Go notation.
func formatHex(u uint64) string {
var f string
switch {
case u <= 0xff:
f = "0x%02x"
case u <= 0xffff:
f = "0x%04x"
case u <= 0xffffff:
f = "0x%06x"
case u <= 0xffffffff:
f = "0x%08x"
case u <= 0xffffffffff:
f = "0x%010x"
case u <= 0xffffffffffff:
f = "0x%012x"
case u <= 0xffffffffffffff:
f = "0x%014x"
case u <= 0xffffffffffffffff:
f = "0x%016x"
}
return fmt.Sprintf(f, u)
}

View File

@@ -0,0 +1,614 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"bytes"
"fmt"
"math"
"reflect"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/google/go-cmp/cmp/internal/diff"
)
// CanFormatDiffSlice reports whether we support custom formatting for nodes
// that are slices of primitive kinds or strings.
func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
switch {
case opts.DiffMode != diffUnknown:
return false // Must be formatting in diff mode
case v.NumDiff == 0:
return false // No differences detected
case !v.ValueX.IsValid() || !v.ValueY.IsValid():
return false // Both values must be valid
case v.NumIgnored > 0:
return false // Some ignore option was used
case v.NumTransformed > 0:
return false // Some transform option was used
case v.NumCompared > 1:
return false // More than one comparison was used
case v.NumCompared == 1 && v.Type.Name() != "":
// The need for cmp to check applicability of options on every element
// in a slice is a significant performance detriment for large []byte.
// The workaround is to specify Comparer(bytes.Equal),
// which enables cmp to compare []byte more efficiently.
// If they differ, we still want to provide batched diffing.
// The logic disallows named types since they tend to have their own
// String method, with nicer formatting than what this provides.
return false
}
// Check whether this is an interface with the same concrete types.
t := v.Type
vx, vy := v.ValueX, v.ValueY
if t.Kind() == reflect.Interface && !vx.IsNil() && !vy.IsNil() && vx.Elem().Type() == vy.Elem().Type() {
vx, vy = vx.Elem(), vy.Elem()
t = vx.Type()
}
// Check whether we provide specialized diffing for this type.
switch t.Kind() {
case reflect.String:
case reflect.Array, reflect.Slice:
// Only slices of primitive types have specialized handling.
switch t.Elem().Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
reflect.Bool, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
default:
return false
}
// Both slice values have to be non-empty.
if t.Kind() == reflect.Slice && (vx.Len() == 0 || vy.Len() == 0) {
return false
}
// If a sufficient number of elements already differ,
// use specialized formatting even if length requirement is not met.
if v.NumDiff > v.NumSame {
return true
}
default:
return false
}
// Use specialized string diffing for longer slices or strings.
const minLength = 32
return vx.Len() >= minLength && vy.Len() >= minLength
}
// FormatDiffSlice prints a diff for the slices (or strings) represented by v.
// This provides custom-tailored logic to make printing of differences in
// textual strings and slices of primitive kinds more readable.
func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
assert(opts.DiffMode == diffUnknown)
t, vx, vy := v.Type, v.ValueX, v.ValueY
if t.Kind() == reflect.Interface {
vx, vy = vx.Elem(), vy.Elem()
t = vx.Type()
opts = opts.WithTypeMode(emitType)
}
// Auto-detect the type of the data.
var sx, sy string
var ssx, ssy []string
var isString, isMostlyText, isPureLinedText, isBinary bool
switch {
case t.Kind() == reflect.String:
sx, sy = vx.String(), vy.String()
isString = true
case t.Kind() == reflect.Slice && t.Elem() == byteType:
sx, sy = string(vx.Bytes()), string(vy.Bytes())
isString = true
case t.Kind() == reflect.Array:
// Arrays need to be addressable for slice operations to work.
vx2, vy2 := reflect.New(t).Elem(), reflect.New(t).Elem()
vx2.Set(vx)
vy2.Set(vy)
vx, vy = vx2, vy2
}
if isString {
var numTotalRunes, numValidRunes, numLines, lastLineIdx, maxLineLen int
for i, r := range sx + sy {
numTotalRunes++
if (unicode.IsPrint(r) || unicode.IsSpace(r)) && r != utf8.RuneError {
numValidRunes++
}
if r == '\n' {
if maxLineLen < i-lastLineIdx {
maxLineLen = i - lastLineIdx
}
lastLineIdx = i + 1
numLines++
}
}
isPureText := numValidRunes == numTotalRunes
isMostlyText = float64(numValidRunes) > math.Floor(0.90*float64(numTotalRunes))
isPureLinedText = isPureText && numLines >= 4 && maxLineLen <= 1024
isBinary = !isMostlyText
// Avoid diffing by lines if it produces a significantly more complex
// edit script than diffing by bytes.
if isPureLinedText {
ssx = strings.Split(sx, "\n")
ssy = strings.Split(sy, "\n")
esLines := diff.Difference(len(ssx), len(ssy), func(ix, iy int) diff.Result {
return diff.BoolResult(ssx[ix] == ssy[iy])
})
esBytes := diff.Difference(len(sx), len(sy), func(ix, iy int) diff.Result {
return diff.BoolResult(sx[ix] == sy[iy])
})
efficiencyLines := float64(esLines.Dist()) / float64(len(esLines))
efficiencyBytes := float64(esBytes.Dist()) / float64(len(esBytes))
quotedLength := len(strconv.Quote(sx + sy))
unquotedLength := len(sx) + len(sy)
escapeExpansionRatio := float64(quotedLength) / float64(unquotedLength)
isPureLinedText = efficiencyLines < 4*efficiencyBytes || escapeExpansionRatio > 1.1
}
}
// Format the string into printable records.
var list textList
var delim string
switch {
// If the text appears to be multi-lined text,
// then perform differencing across individual lines.
case isPureLinedText:
list = opts.formatDiffSlice(
reflect.ValueOf(ssx), reflect.ValueOf(ssy), 1, "line",
func(v reflect.Value, d diffMode) textRecord {
s := formatString(v.Index(0).String())
return textRecord{Diff: d, Value: textLine(s)}
},
)
delim = "\n"
// If possible, use a custom triple-quote (""") syntax for printing
// differences in a string literal. This format is more readable,
// but has edge-cases where differences are visually indistinguishable.
// This format is avoided under the following conditions:
// - A line starts with `"""`
// - A line starts with "..."
// - A line contains non-printable characters
// - Adjacent different lines differ only by whitespace
//
// For example:
//
// """
// ... // 3 identical lines
// foo
// bar
// - baz
// + BAZ
// """
isTripleQuoted := true
prevRemoveLines := map[string]bool{}
prevInsertLines := map[string]bool{}
var list2 textList
list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
for _, r := range list {
if !r.Value.Equal(textEllipsis) {
line, _ := strconv.Unquote(string(r.Value.(textLine)))
line = strings.TrimPrefix(strings.TrimSuffix(line, "\r"), "\r") // trim leading/trailing carriage returns for legacy Windows endline support
normLine := strings.Map(func(r rune) rune {
if unicode.IsSpace(r) {
return -1 // drop whitespace to avoid visually indistinguishable output
}
return r
}, line)
isPrintable := func(r rune) bool {
return unicode.IsPrint(r) || r == '\t' // specially treat tab as printable
}
isTripleQuoted = !strings.HasPrefix(line, `"""`) && !strings.HasPrefix(line, "...") && strings.TrimFunc(line, isPrintable) == ""
switch r.Diff {
case diffRemoved:
isTripleQuoted = isTripleQuoted && !prevInsertLines[normLine]
prevRemoveLines[normLine] = true
case diffInserted:
isTripleQuoted = isTripleQuoted && !prevRemoveLines[normLine]
prevInsertLines[normLine] = true
}
if !isTripleQuoted {
break
}
r.Value = textLine(line)
r.ElideComma = true
}
if !(r.Diff == diffRemoved || r.Diff == diffInserted) { // start a new non-adjacent difference group
prevRemoveLines = map[string]bool{}
prevInsertLines = map[string]bool{}
}
list2 = append(list2, r)
}
if r := list2[len(list2)-1]; r.Diff == diffIdentical && len(r.Value.(textLine)) == 0 {
list2 = list2[:len(list2)-1] // elide single empty line at the end
}
list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
if isTripleQuoted {
var out textNode = &textWrap{Prefix: "(", Value: list2, Suffix: ")"}
switch t.Kind() {
case reflect.String:
if t != stringType {
out = opts.FormatType(t, out)
}
case reflect.Slice:
// Always emit type for slices since the triple-quote syntax
// looks like a string (not a slice).
opts = opts.WithTypeMode(emitType)
out = opts.FormatType(t, out)
}
return out
}
// If the text appears to be single-lined text,
// then perform differencing in approximately fixed-sized chunks.
// The output is printed as quoted strings.
case isMostlyText:
list = opts.formatDiffSlice(
reflect.ValueOf(sx), reflect.ValueOf(sy), 64, "byte",
func(v reflect.Value, d diffMode) textRecord {
s := formatString(v.String())
return textRecord{Diff: d, Value: textLine(s)}
},
)
// If the text appears to be binary data,
// then perform differencing in approximately fixed-sized chunks.
// The output is inspired by hexdump.
case isBinary:
list = opts.formatDiffSlice(
reflect.ValueOf(sx), reflect.ValueOf(sy), 16, "byte",
func(v reflect.Value, d diffMode) textRecord {
var ss []string
for i := 0; i < v.Len(); i++ {
ss = append(ss, formatHex(v.Index(i).Uint()))
}
s := strings.Join(ss, ", ")
comment := commentString(fmt.Sprintf("%c|%v|", d, formatASCII(v.String())))
return textRecord{Diff: d, Value: textLine(s), Comment: comment}
},
)
// For all other slices of primitive types,
// then perform differencing in approximately fixed-sized chunks.
// The size of each chunk depends on the width of the element kind.
default:
var chunkSize int
if t.Elem().Kind() == reflect.Bool {
chunkSize = 16
} else {
switch t.Elem().Bits() {
case 8:
chunkSize = 16
case 16:
chunkSize = 12
case 32:
chunkSize = 8
default:
chunkSize = 8
}
}
list = opts.formatDiffSlice(
vx, vy, chunkSize, t.Elem().Kind().String(),
func(v reflect.Value, d diffMode) textRecord {
var ss []string
for i := 0; i < v.Len(); i++ {
switch t.Elem().Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
ss = append(ss, fmt.Sprint(v.Index(i).Int()))
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
ss = append(ss, fmt.Sprint(v.Index(i).Uint()))
case reflect.Uint8, reflect.Uintptr:
ss = append(ss, formatHex(v.Index(i).Uint()))
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
ss = append(ss, fmt.Sprint(v.Index(i).Interface()))
}
}
s := strings.Join(ss, ", ")
return textRecord{Diff: d, Value: textLine(s)}
},
)
}
// Wrap the output with appropriate type information.
var out textNode = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
if !isMostlyText {
// The "{...}" byte-sequence literal is not valid Go syntax for strings.
// Emit the type for extra clarity (e.g. "string{...}").
if t.Kind() == reflect.String {
opts = opts.WithTypeMode(emitType)
}
return opts.FormatType(t, out)
}
switch t.Kind() {
case reflect.String:
out = &textWrap{Prefix: "strings.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
if t != stringType {
out = opts.FormatType(t, out)
}
case reflect.Slice:
out = &textWrap{Prefix: "bytes.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
if t != bytesType {
out = opts.FormatType(t, out)
}
}
return out
}
// formatASCII formats s as an ASCII string.
// This is useful for printing binary strings in a semi-legible way.
func formatASCII(s string) string {
b := bytes.Repeat([]byte{'.'}, len(s))
for i := 0; i < len(s); i++ {
if ' ' <= s[i] && s[i] <= '~' {
b[i] = s[i]
}
}
return string(b)
}
func (opts formatOptions) formatDiffSlice(
vx, vy reflect.Value, chunkSize int, name string,
makeRec func(reflect.Value, diffMode) textRecord,
) (list textList) {
eq := func(ix, iy int) bool {
return vx.Index(ix).Interface() == vy.Index(iy).Interface()
}
es := diff.Difference(vx.Len(), vy.Len(), func(ix, iy int) diff.Result {
return diff.BoolResult(eq(ix, iy))
})
appendChunks := func(v reflect.Value, d diffMode) int {
n0 := v.Len()
for v.Len() > 0 {
n := chunkSize
if n > v.Len() {
n = v.Len()
}
list = append(list, makeRec(v.Slice(0, n), d))
v = v.Slice(n, v.Len())
}
return n0 - v.Len()
}
var numDiffs int
maxLen := -1
if opts.LimitVerbosity {
maxLen = (1 << opts.verbosity()) << 2 // 4, 8, 16, 32, 64, etc...
opts.VerbosityLevel--
}
groups := coalesceAdjacentEdits(name, es)
groups = coalesceInterveningIdentical(groups, chunkSize/4)
groups = cleanupSurroundingIdentical(groups, eq)
maxGroup := diffStats{Name: name}
for i, ds := range groups {
if maxLen >= 0 && numDiffs >= maxLen {
maxGroup = maxGroup.Append(ds)
continue
}
// Print equal.
if ds.NumDiff() == 0 {
// Compute the number of leading and trailing equal bytes to print.
var numLo, numHi int
numEqual := ds.NumIgnored + ds.NumIdentical
for numLo < chunkSize*numContextRecords && numLo+numHi < numEqual && i != 0 {
numLo++
}
for numHi < chunkSize*numContextRecords && numLo+numHi < numEqual && i != len(groups)-1 {
numHi++
}
if numEqual-(numLo+numHi) <= chunkSize && ds.NumIgnored == 0 {
numHi = numEqual - numLo // Avoid pointless coalescing of single equal row
}
// Print the equal bytes.
appendChunks(vx.Slice(0, numLo), diffIdentical)
if numEqual > numLo+numHi {
ds.NumIdentical -= numLo + numHi
list.AppendEllipsis(ds)
}
appendChunks(vx.Slice(numEqual-numHi, numEqual), diffIdentical)
vx = vx.Slice(numEqual, vx.Len())
vy = vy.Slice(numEqual, vy.Len())
continue
}
// Print unequal.
len0 := len(list)
nx := appendChunks(vx.Slice(0, ds.NumIdentical+ds.NumRemoved+ds.NumModified), diffRemoved)
vx = vx.Slice(nx, vx.Len())
ny := appendChunks(vy.Slice(0, ds.NumIdentical+ds.NumInserted+ds.NumModified), diffInserted)
vy = vy.Slice(ny, vy.Len())
numDiffs += len(list) - len0
}
if maxGroup.IsZero() {
assert(vx.Len() == 0 && vy.Len() == 0)
} else {
list.AppendEllipsis(maxGroup)
}
return list
}
// coalesceAdjacentEdits coalesces the list of edits into groups of adjacent
// equal or unequal counts.
//
// Example:
//
// Input: "..XXY...Y"
// Output: [
// {NumIdentical: 2},
// {NumRemoved: 2, NumInserted 1},
// {NumIdentical: 3},
// {NumInserted: 1},
// ]
func coalesceAdjacentEdits(name string, es diff.EditScript) (groups []diffStats) {
var prevMode byte
lastStats := func(mode byte) *diffStats {
if prevMode != mode {
groups = append(groups, diffStats{Name: name})
prevMode = mode
}
return &groups[len(groups)-1]
}
for _, e := range es {
switch e {
case diff.Identity:
lastStats('=').NumIdentical++
case diff.UniqueX:
lastStats('!').NumRemoved++
case diff.UniqueY:
lastStats('!').NumInserted++
case diff.Modified:
lastStats('!').NumModified++
}
}
return groups
}
// coalesceInterveningIdentical coalesces sufficiently short (<= windowSize)
// equal groups into adjacent unequal groups that currently result in a
// dual inserted/removed printout. This acts as a high-pass filter to smooth
// out high-frequency changes within the windowSize.
//
// Example:
//
// WindowSize: 16,
// Input: [
// {NumIdentical: 61}, // group 0
// {NumRemoved: 3, NumInserted: 1}, // group 1
// {NumIdentical: 6}, // ├── coalesce
// {NumInserted: 2}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 9}, // └── coalesce
// {NumIdentical: 64}, // group 2
// {NumRemoved: 3, NumInserted: 1}, // group 3
// {NumIdentical: 6}, // ├── coalesce
// {NumInserted: 2}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 7}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 2}, // └── coalesce
// {NumIdentical: 63}, // group 4
// ]
// Output: [
// {NumIdentical: 61},
// {NumIdentical: 7, NumRemoved: 12, NumInserted: 3},
// {NumIdentical: 64},
// {NumIdentical: 8, NumRemoved: 12, NumInserted: 3},
// {NumIdentical: 63},
// ]
func coalesceInterveningIdentical(groups []diffStats, windowSize int) []diffStats {
groups, groupsOrig := groups[:0], groups
for i, ds := range groupsOrig {
if len(groups) >= 2 && ds.NumDiff() > 0 {
prev := &groups[len(groups)-2] // Unequal group
curr := &groups[len(groups)-1] // Equal group
next := &groupsOrig[i] // Unequal group
hadX, hadY := prev.NumRemoved > 0, prev.NumInserted > 0
hasX, hasY := next.NumRemoved > 0, next.NumInserted > 0
if ((hadX || hasX) && (hadY || hasY)) && curr.NumIdentical <= windowSize {
*prev = prev.Append(*curr).Append(*next)
groups = groups[:len(groups)-1] // Truncate off equal group
continue
}
}
groups = append(groups, ds)
}
return groups
}
// cleanupSurroundingIdentical scans through all unequal groups, and
// moves any leading sequence of equal elements to the preceding equal group and
// moves and trailing sequence of equal elements to the succeeding equal group.
//
// This is necessary since coalesceInterveningIdentical may coalesce edit groups
// together such that leading/trailing spans of equal elements becomes possible.
// Note that this can occur even with an optimal diffing algorithm.
//
// Example:
//
// Input: [
// {NumIdentical: 61},
// {NumIdentical: 1 , NumRemoved: 11, NumInserted: 2}, // assume 3 leading identical elements
// {NumIdentical: 67},
// {NumIdentical: 7, NumRemoved: 12, NumInserted: 3}, // assume 10 trailing identical elements
// {NumIdentical: 54},
// ]
// Output: [
// {NumIdentical: 64}, // incremented by 3
// {NumRemoved: 9},
// {NumIdentical: 67},
// {NumRemoved: 9},
// {NumIdentical: 64}, // incremented by 10
// ]
func cleanupSurroundingIdentical(groups []diffStats, eq func(i, j int) bool) []diffStats {
var ix, iy int // indexes into sequence x and y
for i, ds := range groups {
// Handle equal group.
if ds.NumDiff() == 0 {
ix += ds.NumIdentical
iy += ds.NumIdentical
continue
}
// Handle unequal group.
nx := ds.NumIdentical + ds.NumRemoved + ds.NumModified
ny := ds.NumIdentical + ds.NumInserted + ds.NumModified
var numLeadingIdentical, numTrailingIdentical int
for j := 0; j < nx && j < ny && eq(ix+j, iy+j); j++ {
numLeadingIdentical++
}
for j := 0; j < nx && j < ny && eq(ix+nx-1-j, iy+ny-1-j); j++ {
numTrailingIdentical++
}
if numIdentical := numLeadingIdentical + numTrailingIdentical; numIdentical > 0 {
if numLeadingIdentical > 0 {
// Remove leading identical span from this group and
// insert it into the preceding group.
if i-1 >= 0 {
groups[i-1].NumIdentical += numLeadingIdentical
} else {
// No preceding group exists, so prepend a new group,
// but do so after we finish iterating over all groups.
defer func() {
groups = append([]diffStats{{Name: groups[0].Name, NumIdentical: numLeadingIdentical}}, groups...)
}()
}
// Increment indexes since the preceding group would have handled this.
ix += numLeadingIdentical
iy += numLeadingIdentical
}
if numTrailingIdentical > 0 {
// Remove trailing identical span from this group and
// insert it into the succeeding group.
if i+1 < len(groups) {
groups[i+1].NumIdentical += numTrailingIdentical
} else {
// No succeeding group exists, so append a new group,
// but do so after we finish iterating over all groups.
defer func() {
groups = append(groups, diffStats{Name: groups[len(groups)-1].Name, NumIdentical: numTrailingIdentical})
}()
}
// Do not increment indexes since the succeeding group will handle this.
}
// Update this group since some identical elements were removed.
nx -= numIdentical
ny -= numIdentical
groups[i] = diffStats{Name: ds.Name, NumRemoved: nx, NumInserted: ny}
}
ix += nx
iy += ny
}
return groups
}

View File

@@ -0,0 +1,432 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import (
"bytes"
"fmt"
"math/rand"
"strings"
"time"
"unicode/utf8"
"github.com/google/go-cmp/cmp/internal/flags"
)
var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
const maxColumnLength = 80
type indentMode int
func (n indentMode) appendIndent(b []byte, d diffMode) []byte {
// The output of Diff is documented as being unstable to provide future
// flexibility in changing the output for more humanly readable reports.
// This logic intentionally introduces instability to the exact output
// so that users can detect accidental reliance on stability early on,
// rather than much later when an actual change to the format occurs.
if flags.Deterministic || randBool {
// Use regular spaces (U+0020).
switch d {
case diffUnknown, diffIdentical:
b = append(b, " "...)
case diffRemoved:
b = append(b, "- "...)
case diffInserted:
b = append(b, "+ "...)
}
} else {
// Use non-breaking spaces (U+00a0).
switch d {
case diffUnknown, diffIdentical:
b = append(b, "  "...)
case diffRemoved:
b = append(b, "- "...)
case diffInserted:
b = append(b, "+ "...)
}
}
return repeatCount(n).appendChar(b, '\t')
}
type repeatCount int
func (n repeatCount) appendChar(b []byte, c byte) []byte {
for ; n > 0; n-- {
b = append(b, c)
}
return b
}
// textNode is a simplified tree-based representation of structured text.
// Possible node types are textWrap, textList, or textLine.
type textNode interface {
// Len reports the length in bytes of a single-line version of the tree.
// Nested textRecord.Diff and textRecord.Comment fields are ignored.
Len() int
// Equal reports whether the two trees are structurally identical.
// Nested textRecord.Diff and textRecord.Comment fields are compared.
Equal(textNode) bool
// String returns the string representation of the text tree.
// It is not guaranteed that len(x.String()) == x.Len(),
// nor that x.String() == y.String() implies that x.Equal(y).
String() string
// formatCompactTo formats the contents of the tree as a single-line string
// to the provided buffer. Any nested textRecord.Diff and textRecord.Comment
// fields are ignored.
//
// However, not all nodes in the tree should be collapsed as a single-line.
// If a node can be collapsed as a single-line, it is replaced by a textLine
// node. Since the top-level node cannot replace itself, this also returns
// the current node itself.
//
// This does not mutate the receiver.
formatCompactTo([]byte, diffMode) ([]byte, textNode)
// formatExpandedTo formats the contents of the tree as a multi-line string
// to the provided buffer. In order for column alignment to operate well,
// formatCompactTo must be called before calling formatExpandedTo.
formatExpandedTo([]byte, diffMode, indentMode) []byte
}
// textWrap is a wrapper that concatenates a prefix and/or a suffix
// to the underlying node.
type textWrap struct {
Prefix string // e.g., "bytes.Buffer{"
Value textNode // textWrap | textList | textLine
Suffix string // e.g., "}"
Metadata interface{} // arbitrary metadata; has no effect on formatting
}
func (s *textWrap) Len() int {
return len(s.Prefix) + s.Value.Len() + len(s.Suffix)
}
func (s1 *textWrap) Equal(s2 textNode) bool {
if s2, ok := s2.(*textWrap); ok {
return s1.Prefix == s2.Prefix && s1.Value.Equal(s2.Value) && s1.Suffix == s2.Suffix
}
return false
}
func (s *textWrap) String() string {
var d diffMode
var n indentMode
_, s2 := s.formatCompactTo(nil, d)
b := n.appendIndent(nil, d) // Leading indent
b = s2.formatExpandedTo(b, d, n) // Main body
b = append(b, '\n') // Trailing newline
return string(b)
}
func (s *textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
n0 := len(b) // Original buffer length
b = append(b, s.Prefix...)
b, s.Value = s.Value.formatCompactTo(b, d)
b = append(b, s.Suffix...)
if _, ok := s.Value.(textLine); ok {
return b, textLine(b[n0:])
}
return b, s
}
func (s *textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
b = append(b, s.Prefix...)
b = s.Value.formatExpandedTo(b, d, n)
b = append(b, s.Suffix...)
return b
}
// textList is a comma-separated list of textWrap or textLine nodes.
// The list may be formatted as multi-lines or single-line at the discretion
// of the textList.formatCompactTo method.
type textList []textRecord
type textRecord struct {
Diff diffMode // e.g., 0 or '-' or '+'
Key string // e.g., "MyField"
Value textNode // textWrap | textLine
ElideComma bool // avoid trailing comma
Comment fmt.Stringer // e.g., "6 identical fields"
}
// AppendEllipsis appends a new ellipsis node to the list if none already
// exists at the end. If cs is non-zero it coalesces the statistics with the
// previous diffStats.
func (s *textList) AppendEllipsis(ds diffStats) {
hasStats := !ds.IsZero()
if len(*s) == 0 || !(*s)[len(*s)-1].Value.Equal(textEllipsis) {
if hasStats {
*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true, Comment: ds})
} else {
*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true})
}
return
}
if hasStats {
(*s)[len(*s)-1].Comment = (*s)[len(*s)-1].Comment.(diffStats).Append(ds)
}
}
func (s textList) Len() (n int) {
for i, r := range s {
n += len(r.Key)
if r.Key != "" {
n += len(": ")
}
n += r.Value.Len()
if i < len(s)-1 {
n += len(", ")
}
}
return n
}
func (s1 textList) Equal(s2 textNode) bool {
if s2, ok := s2.(textList); ok {
if len(s1) != len(s2) {
return false
}
for i := range s1 {
r1, r2 := s1[i], s2[i]
if !(r1.Diff == r2.Diff && r1.Key == r2.Key && r1.Value.Equal(r2.Value) && r1.Comment == r2.Comment) {
return false
}
}
return true
}
return false
}
func (s textList) String() string {
return (&textWrap{Prefix: "{", Value: s, Suffix: "}"}).String()
}
func (s textList) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
s = append(textList(nil), s...) // Avoid mutating original
// Determine whether we can collapse this list as a single line.
n0 := len(b) // Original buffer length
var multiLine bool
for i, r := range s {
if r.Diff == diffInserted || r.Diff == diffRemoved {
multiLine = true
}
b = append(b, r.Key...)
if r.Key != "" {
b = append(b, ": "...)
}
b, s[i].Value = r.Value.formatCompactTo(b, d|r.Diff)
if _, ok := s[i].Value.(textLine); !ok {
multiLine = true
}
if r.Comment != nil {
multiLine = true
}
if i < len(s)-1 {
b = append(b, ", "...)
}
}
// Force multi-lined output when printing a removed/inserted node that
// is sufficiently long.
if (d == diffInserted || d == diffRemoved) && len(b[n0:]) > maxColumnLength {
multiLine = true
}
if !multiLine {
return b, textLine(b[n0:])
}
return b, s
}
func (s textList) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
alignKeyLens := s.alignLens(
func(r textRecord) bool {
_, isLine := r.Value.(textLine)
return r.Key == "" || !isLine
},
func(r textRecord) int { return utf8.RuneCountInString(r.Key) },
)
alignValueLens := s.alignLens(
func(r textRecord) bool {
_, isLine := r.Value.(textLine)
return !isLine || r.Value.Equal(textEllipsis) || r.Comment == nil
},
func(r textRecord) int { return utf8.RuneCount(r.Value.(textLine)) },
)
// Format lists of simple lists in a batched form.
// If the list is sequence of only textLine values,
// then batch multiple values on a single line.
var isSimple bool
for _, r := range s {
_, isLine := r.Value.(textLine)
isSimple = r.Diff == 0 && r.Key == "" && isLine && r.Comment == nil
if !isSimple {
break
}
}
if isSimple {
n++
var batch []byte
emitBatch := func() {
if len(batch) > 0 {
b = n.appendIndent(append(b, '\n'), d)
b = append(b, bytes.TrimRight(batch, " ")...)
batch = batch[:0]
}
}
for _, r := range s {
line := r.Value.(textLine)
if len(batch)+len(line)+len(", ") > maxColumnLength {
emitBatch()
}
batch = append(batch, line...)
batch = append(batch, ", "...)
}
emitBatch()
n--
return n.appendIndent(append(b, '\n'), d)
}
// Format the list as a multi-lined output.
n++
for i, r := range s {
b = n.appendIndent(append(b, '\n'), d|r.Diff)
if r.Key != "" {
b = append(b, r.Key+": "...)
}
b = alignKeyLens[i].appendChar(b, ' ')
b = r.Value.formatExpandedTo(b, d|r.Diff, n)
if !r.ElideComma {
b = append(b, ',')
}
b = alignValueLens[i].appendChar(b, ' ')
if r.Comment != nil {
b = append(b, " // "+r.Comment.String()...)
}
}
n--
return n.appendIndent(append(b, '\n'), d)
}
func (s textList) alignLens(
skipFunc func(textRecord) bool,
lenFunc func(textRecord) int,
) []repeatCount {
var startIdx, endIdx, maxLen int
lens := make([]repeatCount, len(s))
for i, r := range s {
if skipFunc(r) {
for j := startIdx; j < endIdx && j < len(s); j++ {
lens[j] = repeatCount(maxLen - lenFunc(s[j]))
}
startIdx, endIdx, maxLen = i+1, i+1, 0
} else {
if maxLen < lenFunc(r) {
maxLen = lenFunc(r)
}
endIdx = i + 1
}
}
for j := startIdx; j < endIdx && j < len(s); j++ {
lens[j] = repeatCount(maxLen - lenFunc(s[j]))
}
return lens
}
// textLine is a single-line segment of text and is always a leaf node
// in the textNode tree.
type textLine []byte
var (
textNil = textLine("nil")
textEllipsis = textLine("...")
)
func (s textLine) Len() int {
return len(s)
}
func (s1 textLine) Equal(s2 textNode) bool {
if s2, ok := s2.(textLine); ok {
return bytes.Equal([]byte(s1), []byte(s2))
}
return false
}
func (s textLine) String() string {
return string(s)
}
func (s textLine) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
return append(b, s...), s
}
func (s textLine) formatExpandedTo(b []byte, _ diffMode, _ indentMode) []byte {
return append(b, s...)
}
type diffStats struct {
Name string
NumIgnored int
NumIdentical int
NumRemoved int
NumInserted int
NumModified int
}
func (s diffStats) IsZero() bool {
s.Name = ""
return s == diffStats{}
}
func (s diffStats) NumDiff() int {
return s.NumRemoved + s.NumInserted + s.NumModified
}
func (s diffStats) Append(ds diffStats) diffStats {
assert(s.Name == ds.Name)
s.NumIgnored += ds.NumIgnored
s.NumIdentical += ds.NumIdentical
s.NumRemoved += ds.NumRemoved
s.NumInserted += ds.NumInserted
s.NumModified += ds.NumModified
return s
}
// String prints a humanly-readable summary of coalesced records.
//
// Example:
//
// diffStats{Name: "Field", NumIgnored: 5}.String() => "5 ignored fields"
func (s diffStats) String() string {
var ss []string
var sum int
labels := [...]string{"ignored", "identical", "removed", "inserted", "modified"}
counts := [...]int{s.NumIgnored, s.NumIdentical, s.NumRemoved, s.NumInserted, s.NumModified}
for i, n := range counts {
if n > 0 {
ss = append(ss, fmt.Sprintf("%d %v", n, labels[i]))
}
sum += n
}
// Pluralize the name (adjusting for some obscure English grammar rules).
name := s.Name
if sum > 1 {
name += "s"
if strings.HasSuffix(name, "ys") {
name = name[:len(name)-2] + "ies" // e.g., "entrys" => "entries"
}
}
// Format the list according to English grammar (with Oxford comma).
switch n := len(ss); n {
case 0:
return ""
case 1, 2:
return strings.Join(ss, " and ") + " " + name
default:
return strings.Join(ss[:n-1], ", ") + ", and " + ss[n-1] + " " + name
}
}
type commentString string
func (s commentString) String() string { return string(s) }

View File

@@ -0,0 +1,121 @@
// Copyright 2019, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cmp
import "reflect"
// valueNode represents a single node within a report, which is a
// structured representation of the value tree, containing information
// regarding which nodes are equal or not.
type valueNode struct {
parent *valueNode
Type reflect.Type
ValueX reflect.Value
ValueY reflect.Value
// NumSame is the number of leaf nodes that are equal.
// All descendants are equal only if NumDiff is 0.
NumSame int
// NumDiff is the number of leaf nodes that are not equal.
NumDiff int
// NumIgnored is the number of leaf nodes that are ignored.
NumIgnored int
// NumCompared is the number of leaf nodes that were compared
// using an Equal method or Comparer function.
NumCompared int
// NumTransformed is the number of non-leaf nodes that were transformed.
NumTransformed int
// NumChildren is the number of transitive descendants of this node.
// This counts from zero; thus, leaf nodes have no descendants.
NumChildren int
// MaxDepth is the maximum depth of the tree. This counts from zero;
// thus, leaf nodes have a depth of zero.
MaxDepth int
// Records is a list of struct fields, slice elements, or map entries.
Records []reportRecord // If populated, implies Value is not populated
// Value is the result of a transformation, pointer indirect, of
// type assertion.
Value *valueNode // If populated, implies Records is not populated
// TransformerName is the name of the transformer.
TransformerName string // If non-empty, implies Value is populated
}
type reportRecord struct {
Key reflect.Value // Invalid for slice element
Value *valueNode
}
func (parent *valueNode) PushStep(ps PathStep) (child *valueNode) {
vx, vy := ps.Values()
child = &valueNode{parent: parent, Type: ps.Type(), ValueX: vx, ValueY: vy}
switch s := ps.(type) {
case StructField:
assert(parent.Value == nil)
parent.Records = append(parent.Records, reportRecord{Key: reflect.ValueOf(s.Name()), Value: child})
case SliceIndex:
assert(parent.Value == nil)
parent.Records = append(parent.Records, reportRecord{Value: child})
case MapIndex:
assert(parent.Value == nil)
parent.Records = append(parent.Records, reportRecord{Key: s.Key(), Value: child})
case Indirect:
assert(parent.Value == nil && parent.Records == nil)
parent.Value = child
case TypeAssertion:
assert(parent.Value == nil && parent.Records == nil)
parent.Value = child
case Transform:
assert(parent.Value == nil && parent.Records == nil)
parent.Value = child
parent.TransformerName = s.Name()
parent.NumTransformed++
default:
assert(parent == nil) // Must be the root step
}
return child
}
func (r *valueNode) Report(rs Result) {
assert(r.MaxDepth == 0) // May only be called on leaf nodes
if rs.ByIgnore() {
r.NumIgnored++
} else {
if rs.Equal() {
r.NumSame++
} else {
r.NumDiff++
}
}
assert(r.NumSame+r.NumDiff+r.NumIgnored == 1)
if rs.ByMethod() {
r.NumCompared++
}
if rs.ByFunc() {
r.NumCompared++
}
assert(r.NumCompared <= 1)
}
func (child *valueNode) PopStep() (parent *valueNode) {
if child.parent == nil {
return nil
}
parent = child.parent
parent.NumSame += child.NumSame
parent.NumDiff += child.NumDiff
parent.NumIgnored += child.NumIgnored
parent.NumCompared += child.NumCompared
parent.NumTransformed += child.NumTransformed
parent.NumChildren += child.NumChildren + 1
if parent.MaxDepth < child.MaxDepth+1 {
parent.MaxDepth = child.MaxDepth + 1
}
return parent
}

View File

@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

View File

@@ -0,0 +1,57 @@
version: 2
builds:
-
id: "cpuid"
binary: cpuid
main: ./cmd/cpuid/main.go
env:
- CGO_ENABLED=0
flags:
- -ldflags=-s -w
goos:
- aix
- linux
- freebsd
- netbsd
- windows
- darwin
goarch:
- 386
- amd64
- arm64
goarm:
- 7
archives:
-
id: cpuid
name_template: "cpuid-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: zip
files:
- LICENSE
checksum:
name_template: 'checksums.txt'
changelog:
sort: asc
filters:
exclude:
- '^doc:'
- '^docs:'
- '^test:'
- '^tests:'
- '^Update\sREADME.md'
nfpms:
-
file_name_template: "cpuid_package_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
vendor: Klaus Post
homepage: https://github.com/klauspost/cpuid
maintainer: Klaus Post <klauspost@gmail.com>
description: CPUID Tool
license: BSD 3-Clause
formats:
- deb
- rpm

View File

@@ -0,0 +1,35 @@
Developer Certificate of Origin
Version 1.1
Copyright (C) 2015- Klaus Post & Contributors.
Email: klauspost@gmail.com
Everyone is permitted to copy and distribute verbatim copies of this
license document, but changing it is not allowed.
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.

Some files were not shown because too many files have changed in this diff Show More