diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6b4a05d0491..a5eb96467bb 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -157,8 +157,25 @@ jobs: git fetch --no-tags --unshallow origin fi + # Coverage for the CI dashboard: on Linux, for every push to main, and for a PR + # whose head commit message has "with-coverage" (to try a change to it). The + # directories are outside the checkout, so that the check for new files below + # does not see them. The PR's head commit is there after the fetch above. + - name: Decide whether to collect coverage + id: cover + if: ${{ matrix.os.name == 'linux' && (github.event_name == 'push' || github.event_name == 'pull_request') }} + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ "$GITHUB_EVENT_NAME" = "push" ] || git log -1 --format=%B "$HEAD_SHA" | grep -q 'with-coverage'; then + echo "unit=$RUNNER_TEMP/cover-unit" >> "$GITHUB_OUTPUT" + echo "acc=$RUNNER_TEMP/cover-acc" >> "$GITHUB_OUTPUT" + fi + - name: Run tests env: + CLI_GOCOVERDIR_UNIT: ${{ steps.cover.outputs.unit }} + CLI_GOCOVERDIR: ${{ steps.cover.outputs.acc }} ENVFILTER: ${{ matrix.deployment.envfilter }} # On pull requests, run only a subset of acceptance subtests on the slower # windows/macOS cells so neither lands on the critical path to merge. An @@ -181,6 +198,32 @@ jobs: if-no-files-found: warn retention-days: 7 + # Failed tests count too: their coverage is as real as that of the ones that passed. + - name: Pack coverage + if: ${{ always() && steps.cover.outputs.acc != '' }} + env: + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OS: ${{ matrix.os.name }} + CELL: ${{ matrix.deployment.name }} + UNIT_DIR: ${{ steps.cover.outputs.unit }} + ACC_DIR: ${{ steps.cover.outputs.acc }} + run: | + go -C tools run ./coverpack -root "$GITHUB_WORKSPACE" -unit "$UNIT_DIR" -acc "$ACC_DIR" \ + -o "$RUNNER_TEMP/coverage.bin.zst" \ + -meta "{\"sha\": \"$SHA\", \"os\": \"$OS\", \"cell\": \"$CELL\", \"pr\": ${PR_NUMBER:-null}}" + + - name: Upload coverage + if: ${{ always() && steps.cover.outputs.acc != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-${{ matrix.os.name }}-${{ matrix.deployment.name }} + path: ${{ runner.temp }}/coverage.bin.zst + if-no-files-found: warn + retention-days: 7 + # Already compressed. + compression-level: 0 + # `task test` prints a lot of output that GitHub is slow to render, so # surface just the failed test names on the job summary page (and in this # step's log) for a quick read of what broke. diff --git a/Taskfile.yml b/Taskfile.yml index 0c4184c9778..38ed47595df 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -458,6 +458,12 @@ tasks: # The slowest unit package observed across Linux/macOS/Windows is ~83s, so 10m is # ample headroom while a hung package (e.g. a wedged goroutine) fails in minutes # instead of burning the full hour. LOCAL_TIMEOUT still overrides when needed. + # + # CLI_GOCOVERDIR_UNIT (an absolute path) turns on coverage of every package by + # every test (tools/coverpack reads it). It is a coverage directory, not + # -coverprofile, because --rerun-fails runs go test again with the same flags and + # that would overwrite the profile with the coverage of the rerun alone. + - '[ -z "$CLI_GOCOVERDIR_UNIT" ] || mkdir -p "$CLI_GOCOVERDIR_UNIT"' - | {{.GO_TOOL}} gotestsum \ --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ @@ -465,7 +471,8 @@ tasks: --jsonfile test-output-unit-root.json \ --rerun-fails \ --packages "{{.TEST_PACKAGES}}" \ - -- -timeout=${LOCAL_TIMEOUT:-10m} + -- ${CLI_GOCOVERDIR_UNIT:+-cover -coverpkg=./...} -timeout=${LOCAL_TIMEOUT:-10m} \ + ${CLI_GOCOVERDIR_UNIT:+-args -test.gocoverdir=$CLI_GOCOVERDIR_UNIT} test-unit-tools: desc: Run unit tests in tools/ module diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 955703d80ef..0ada9a0baf1 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -1012,6 +1012,8 @@ func runTest(t *testing.T, } err := os.MkdirAll(coverDir, os.ModePerm) require.NoError(t, err) + // The directory name does not say which variant it is; tools/coverpack reads the test name from here. + testutil.WriteFile(t, filepath.Join(coverDir, "name"), t.Name()) cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } diff --git a/tools/coverpack/format.go b/tools/coverpack/format.go new file mode 100644 index 00000000000..28198d64c4f --- /dev/null +++ b/tools/coverpack/format.go @@ -0,0 +1,245 @@ +package main + +// The packed file, zstd-compressed. Inside, a uvarint is unsigned LEB128 (encoding/binary's +// uvarint): +// +// "CLICOV1\n" +// uvarint n, n bytes meta, JSON: the -meta object plus "go", "module" and "commit" (the commit that was +// checked out and tested; for a PR, GitHub's merge of it into the base branch) +// uvarint L layouts, sorted by path: +// uvarint n, n bytes path relative to the module ("bundle/deploy/x.go") +// 20 bytes git blob sha of the file as read (zeros if it was not found) +// uvarint line count +// uvarint B blocks, sorted by start, then end; per block: +// uvarint sl - previous sl, uvarint sc, uvarint el - sl, uvarint ec, uvarint statements +// (as in a Go cover profile: lines and byte columns 1-based, the end exclusive) +// L times the distinct bitmaps of each layout, in layout order: +// uvarint P, then P bitmaps of ceil(B / 8) bytes: block i is bit i % 8 of byte i / 8 +// uvarint N profiles: +// 1 byte kind 0: all acceptance tests merged, 1: unit tests merged, 2: one acceptance test +// uvarint n, n bytes test name (t.Name()), empty for kinds 0 and 1 +// uvarint F files, by layout; per file: uvarint layout - previous layout, uvarint bitmap index +// +// Kinds 0 and 1 list every file of their build, covered or not: they are its inventory. A file +// that a test (kind 2) does not list was not covered by it if the build has the file, else it is +// unknown. + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +const magic = "CLICOV1\n" + +const ( + kindAll = 0 + kindUnit = 1 + kindTest = 2 +) + +type layout struct { + path string // relative to the module + sha [20]byte + lines int + blocks []block + first int // catalog id of blocks[0]; the ids of a layout are consecutive +} + +type profile struct { + kind byte + name string + files []int // layouts, ascending + bits bitset +} + +type bitset []uint64 + +func newBitset(n int) bitset { return make(bitset, (n+63)/64) } + +func (b bitset) set(i int32) { b[i>>6] |= 1 << (i & 63) } +func (b bitset) has(i int) bool { return b[i>>6]>>(i&63)&1 != 0 } + +func (b bitset) or(o bitset) { + for i := range b { + b[i] |= o[i] + } +} + +// bitmap is the bits of a layout, in the order of the file. +func (b bitset) bitmap(l *layout) []byte { + out := make([]byte, (len(l.blocks)+7)/8) + for i := range l.blocks { + if b.has(l.first + i) { + out[i/8] |= 1 << (i % 8) + } + } + return out +} + +// encode returns the uncompressed file and the number of distinct bitmaps in it. +func encode(meta []byte, layouts []*layout, profiles []profile) ([]byte, int) { + distinct := make([]map[string]int, len(layouts)) + order := make([][][]byte, len(layouts)) + index := make([][]int, len(profiles)) + for p, prof := range profiles { + for _, l := range prof.files { + bm := prof.bits.bitmap(layouts[l]) + if distinct[l] == nil { + distinct[l] = map[string]int{} + } + i, ok := distinct[l][string(bm)] + if !ok { + i = len(order[l]) + distinct[l][string(bm)] = i + order[l] = append(order[l], bm) + } + index[p] = append(index[p], i) + } + } + + out := []byte(magic) + put := func(v int) { out = binary.AppendUvarint(out, uint64(v)) } + putBytes := func(b []byte) { put(len(b)); out = append(out, b...) } + + putBytes(meta) + put(len(layouts)) + for _, l := range layouts { + putBytes([]byte(l.path)) + out = append(out, l.sha[:]...) + put(l.lines) + put(len(l.blocks)) + prev := uint32(0) + for _, b := range l.blocks { + put(int(b.sl - prev)) + put(int(b.sc)) + put(int(b.el - b.sl)) + put(int(b.ec)) + put(int(b.stmts)) + prev = b.sl + } + } + total := 0 + for l := range layouts { + put(len(order[l])) + for _, bm := range order[l] { + out = append(out, bm...) + } + total += len(order[l]) + } + put(len(profiles)) + for p, prof := range profiles { + out = append(out, prof.kind) + putBytes([]byte(prof.name)) + put(len(prof.files)) + prev := 0 + for i, l := range prof.files { + put(l - prev) + put(index[p][i]) + prev = l + } + } + return out, total +} + +// decoded is a packed file read back. +type decoded struct { + meta []byte + layouts []*layout + bitmaps [][][]byte // per layout + profiles []decodedProfile +} + +type decodedProfile struct { + kind byte + name string + files []int // layouts, ascending + index []int // bitmap of each file +} + +func decode(data []byte) (*decoded, error) { + c := &cursor{b: data} + if string(c.take(len(magic))) != magic { + return nil, errors.New("not a coverpack file") + } + str := func() []byte { return c.take(c.count()) } + d := &decoded{meta: str()} + d.layouts = make([]*layout, c.count()) + for i := range d.layouts { + l := &layout{path: string(str())} + copy(l.sha[:], c.take(20)) + l.lines = int(c.uleb128()) + l.blocks = make([]block, c.count()) + sl := uint32(0) + for j := range l.blocks { + sl += uint32(c.uleb128()) + b := block{file: l.path, sl: sl, sc: uint32(c.uleb128())} + b.el = sl + uint32(c.uleb128()) + b.ec = uint32(c.uleb128()) + b.stmts = uint32(c.uleb128()) + l.blocks[j] = b + } + d.layouts[i] = l + } + d.bitmaps = make([][][]byte, len(d.layouts)) + for i, l := range d.layouts { + d.bitmaps[i] = make([][]byte, c.count()) + for j := range d.bitmaps[i] { + d.bitmaps[i][j] = c.take((len(l.blocks) + 7) / 8) + } + } + d.profiles = make([]decodedProfile, c.count()) + for i := range d.profiles { + p := decodedProfile{kind: c.u8(), name: string(str())} + n, l := c.count(), 0 + for range n { + l += int(c.uleb128()) + p.files = append(p.files, l) + p.index = append(p.index, int(c.uleb128())) + } + d.profiles[i] = p + } + if c.err != nil { + return nil, c.err + } + for _, p := range d.profiles { + for i, l := range p.files { + if l >= len(d.layouts) || p.index[i] >= len(d.bitmaps[l]) { + return nil, errMalformed + } + } + } + return d, nil +} + +// dumpText writes the profiles as Go cover profile text, each after a "# all", "# unit" or +// "# test NAME" line. A test lists its covered blocks only. +func dumpText(w io.Writer, d *decoded, module string) error { + if _, err := fmt.Fprintf(w, "# coverage %s\n", d.meta); err != nil { + return err + } + for _, p := range d.profiles { + header := []string{"# all", "# unit"} + h := "# test " + p.name + if int(p.kind) < len(header) { + h = header[p.kind] + } + if _, err := fmt.Fprintln(w, h); err != nil { + return err + } + for i, l := range p.files { + bm := d.bitmaps[l][p.index[i]] + for j, b := range d.layouts[l].blocks { + covered := bm[j/8] >> (j % 8) & 1 + if covered == 0 && p.kind == kindTest { + continue + } + if _, err := fmt.Fprintf(w, "%s/%s:%d.%d,%d.%d %d %d\n", module, b.file, b.sl, b.sc, b.el, b.ec, b.stmts, covered); err != nil { + return err + } + } + } + } + return nil +} diff --git a/tools/coverpack/govcov.go b/tools/coverpack/govcov.go new file mode 100644 index 00000000000..15f9275d74a --- /dev/null +++ b/tools/coverpack/govcov.go @@ -0,0 +1,299 @@ +package main + +// Readers for the files that a binary built with -cover writes to GOCOVERDIR. The formats are +// those of Go's internal/coverage packages (defs.go, decodemeta, decodecounter), which code +// outside the standard library cannot import: version 1 of both, the only one there has been. + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" +) + +var ( + metaMagic = []byte{0x00, 0x63, 0x76, 0x6d} + counterMagic = []byte{0x00, 0x63, 0x77, 0x6d} + + errTruncated = errors.New("truncated") + errMalformed = errors.New("malformed") +) + +const ( + metaFileVersion = 1 + counterFileVersion = 1 + metaSymbolHeaderSize = 44 + counterFileHeaderSize = 32 + counterFooterSize = 16 + granularityPerBlock = 1 + flavorRaw = 1 + flavorULEB128 = 2 +) + +// cursor reads a byte slice from the front. The first read past the end sets err; reads after +// that return zeros, so a caller checks err once after a run of reads. +type cursor struct { + b []byte + off int + err error +} + +func (c *cursor) take(n int) []byte { + if c.err != nil { + return nil + } + if n < 0 || n > len(c.b)-c.off { + c.err = errTruncated + return nil + } + v := c.b[c.off : c.off+n] + c.off += n + return v +} + +func (c *cursor) seek(off int) { + if c.err == nil && (off < 0 || off > len(c.b)) { + c.err = errTruncated + return + } + c.off = off +} + +func (c *cursor) u8() uint8 { + if b := c.take(1); b != nil { + return b[0] + } + return 0 +} + +func (c *cursor) u32() uint32 { + if b := c.take(4); b != nil { + return binary.LittleEndian.Uint32(b) + } + return 0 +} + +func (c *cursor) u64() uint64 { + if b := c.take(8); b != nil { + return binary.LittleEndian.Uint64(b) + } + return 0 +} + +func (c *cursor) uleb128() uint64 { + var v uint64 + for shift := 0; c.err == nil; shift += 7 { + if c.off >= len(c.b) { + c.err = errTruncated + break + } + b := c.b[c.off] + c.off++ + if shift > 63 { + c.err = errMalformed + break + } + v |= uint64(b&0x7f) << shift + if b < 0x80 { + return v + } + } + return 0 +} + +// count reads a uleb128 count of items that each take at least one byte, so that a corrupt +// count cannot make the caller allocate more than the input could hold. +func (c *cursor) count() int { + n := c.uleb128() + if c.err == nil && n > uint64(len(c.b)-c.off) { + c.err = errMalformed + return 0 + } + return int(n) +} + +// block is a Go coverage unit: a span of statements that run together. Lines and columns are +// 1-based, columns count bytes, and the end is exclusive. +type block struct { + file string // as Go names it: import path of the package, then the file name + sl, sc, el, ec uint32 + stmts uint32 +} + +// pkgMeta is the coverage meta-data of one package: the blocks of each of its functions, in +// the order of the counters. +type pkgMeta struct { + path string + module string + funcs [][]block + ids [][]int32 // the catalog ids of funcs, filled in once the catalog is known +} + +// readMeta returns the packages of a covmeta file, indexed as counter files index them. A +// package is decoded once: cache holds the packages seen so far, by their hash. +func readMeta(data []byte, cache map[[16]byte]*pkgMeta) ([]*pkgMeta, error) { + c := &cursor{b: data} + if !bytes.Equal(c.take(4), metaMagic) { + return nil, errors.New("not a coverage meta-data file") + } + if v := c.u32(); v > metaFileVersion { + return nil, fmt.Errorf("meta-data file version %d, only %d is known", v, metaFileVersion) + } + c.u64() // total length + n := c.u64() + c.take(16 + 4 + 4 + 1) // hash, string table offset and length (only package payloads have strings), counter mode + if g := c.u8(); c.err == nil && g != granularityPerBlock { + return nil, fmt.Errorf("counter granularity %d, only per block is supported", g) + } + c.take(6) + if c.err == nil && n > uint64(len(data))/16 { + return nil, errMalformed + } + offsets := make([]uint64, n) + for i := range offsets { + offsets[i] = c.u64() + } + pkgs := make([]*pkgMeta, n) + for i := range pkgs { + length := c.u64() + if c.err != nil { + break + } + if offsets[i] > uint64(len(data)) || length > uint64(len(data))-offsets[i] { + return nil, errTruncated + } + p, err := readPackage(data[offsets[i]:offsets[i]+length], cache) + if err != nil { + return nil, fmt.Errorf("package %d: %w", i, err) + } + pkgs[i] = p + } + if c.err != nil { + return nil, c.err + } + return pkgs, nil +} + +func readPackage(b []byte, cache map[[16]byte]*pkgMeta) (*pkgMeta, error) { + c := &cursor{b: b} + c.u32() // length + c.u32() // package name + pathIndex, moduleIndex := c.u32(), c.u32() + var hash [16]byte + copy(hash[:], c.take(16)) + c.take(4) + c.u32() // number of files + numFuncs := int(c.u32()) + if c.err != nil { + return nil, c.err + } + if p := cache[hash]; p != nil { + return p, nil + } + + c.seek(metaSymbolHeaderSize + 4*numFuncs) + strs := make([]string, c.count()) + for i := range strs { + strs[i] = string(c.take(c.count())) + } + str := func(i uint64) string { + if i >= uint64(len(strs)) { + if c.err == nil { + c.err = errMalformed + } + return "" + } + return strs[i] + } + + p := &pkgMeta{path: str(uint64(pathIndex)), module: str(uint64(moduleIndex)), funcs: make([][]block, numFuncs)} + for f := range p.funcs { + c.seek(metaSymbolHeaderSize + 4*f) + c.seek(int(c.u32())) + units := make([]block, c.count()) + c.uleb128() // function name + file := str(c.uleb128()) + for u := range units { + units[u] = block{ + file: file, + sl: uint32(c.uleb128()), + sc: uint32(c.uleb128()), + el: uint32(c.uleb128()), + ec: uint32(c.uleb128()), + stmts: uint32(c.uleb128()), + } + } + p.funcs[f] = units + } + if c.err != nil { + return nil, c.err + } + cache[hash] = p + return p, nil +} + +// readCounters calls visit for every function with counters in a covcounters file: the hash +// of its meta-data file, the package index in it, the function index and the counters, one per +// block. The counters slice is reused between calls. +func readCounters(data []byte, visit func(metaHash [16]byte, pkg, fn uint32, counters []uint32)) error { + c := &cursor{b: data} + if !bytes.Equal(c.take(4), counterMagic) { + return errors.New("not a coverage counter file") + } + if v := c.u32(); v > counterFileVersion { + return fmt.Errorf("counter file version %d, only %d is known", v, counterFileVersion) + } + var hash [16]byte + copy(hash[:], c.take(16)) + flavor, bigEndian := c.u8(), c.u8() != 0 + c.take(6) + if c.err != nil || len(data) < counterFileHeaderSize+counterFooterSize { + return errTruncated + } + + footer := &cursor{b: data[len(data)-counterFooterSize:]} + if !bytes.Equal(footer.take(4), counterMagic) { + return errors.New("bad counter file footer") + } + footer.take(4) + segments := footer.u32() + + var value func() uint32 + switch { + case flavor == flavorULEB128: + value = func() uint32 { return uint32(c.uleb128()) } + case flavor == flavorRaw && bigEndian: + value = func() uint32 { + if b := c.take(4); b != nil { + return binary.BigEndian.Uint32(b) + } + return 0 + } + case flavor == flavorRaw: + value = c.u32 + default: + return fmt.Errorf("unknown counter flavor %d", flavor) + } + + counters := make([]uint32, 0, 1024) + for range segments { + funcs := c.u64() + strTabLen, argsLen := c.u32(), c.u32() + c.take(int(strTabLen) + int(argsLen)) // the args (os.Args, GOOS, GOARCH) are not needed + if r := c.off % 4; r != 0 { + c.take(4 - r) + } + for i := uint64(0); i < funcs && c.err == nil; i++ { + n, pkg, fn := value(), value(), value() + counters = counters[:0] + for j := uint32(0); j < n && c.err == nil; j++ { + counters = append(counters, value()) + } + if c.err == nil { + visit(hash, pkg, fn, counters) + } + } + c.take(counterFooterSize) + } + return c.err +} diff --git a/tools/coverpack/main.go b/tools/coverpack/main.go new file mode 100644 index 00000000000..e4d233d24f8 --- /dev/null +++ b/tools/coverpack/main.go @@ -0,0 +1,461 @@ +// Command coverpack packs the coverage of a test run into one small file for the CI dashboard +// (format.go). +// +// It reads the coverage directory of the unit tests (-unit, CLI_GOCOVERDIR_UNIT) and that of +// the acceptance tests (-acc, CLI_GOCOVERDIR: a directory per test, with the test name in its +// "name" file), and decodes Go's binary coverage files itself (govcov.go). `go tool covdata` +// would have to run once per test, and each time parse the CLI's meta-data (about 1 MB, of +// which every test directory has a copy); here each distinct meta-data file is parsed once and +// a test costs one pass over its counter files. +// +// Go measures blocks, which never overlap, so the coverage of a file is one bit per block. +// Most tests cover a file the same way as many other tests (the CLI's startup, mostly), so each +// file stores its distinct bitmaps once and a test names, per file it covered, which of them it +// has. +package main + +import ( + "bufio" + "cmp" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "maps" + "math/bits" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/klauspost/compress/zstd" +) + +func main() { + var o options + flag.StringVar(&o.unitDir, "unit", "", "unit coverage directory (CLI_GOCOVERDIR_UNIT)") + flag.StringVar(&o.accDir, "acc", "", "acceptance coverage directory (CLI_GOCOVERDIR)") + flag.StringVar(&o.root, "root", ".", "root of the module, where the source files are read") + flag.StringVar(&o.meta, "meta", "{}", "JSON object for the meta (sha, os, cell, ...)") + flag.StringVar(&o.output, "o", "", "the .bin.zst to write") + dumpPath := flag.String("dump", "", "print `FILE` as Go cover profile text instead") + flag.Parse() + + var err error + switch { + case *dumpPath != "": + err = dump(os.Stdout, *dumpPath) + case o.output == "": + err = errors.New("-o is required") + default: + err = pack(o) + } + if err != nil { + fmt.Fprintln(os.Stderr, "coverpack:", err) + os.Exit(1) + } +} + +type options struct { + unitDir, accDir, root, meta, output string +} + +// scanDir adds the meta-data files of dir to metas (by their hash, which names them) and +// returns its counter files. +func scanDir(dir string, metas map[[16]byte]string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var counters []string + for _, e := range entries { + name := e.Name() + switch { + case strings.HasPrefix(name, "covmeta."): + h, err := hex.DecodeString(strings.TrimPrefix(name, "covmeta.")) + if err != nil || len(h) != 16 { + return nil, fmt.Errorf("%s: unexpected meta-data file name", filepath.Join(dir, name)) + } + var hash [16]byte + copy(hash[:], h) + if _, ok := metas[hash]; !ok { + metas[hash] = filepath.Join(dir, name) + } + case strings.HasPrefix(name, "covcounters."): + counters = append(counters, filepath.Join(dir, name)) + } + } + return counters, nil +} + +type acceptanceTest struct { + name string + counters []string +} + +// acceptanceTests returns the tests that ran the CLI, sorted by name. +func acceptanceTests(dir string, metas map[[16]byte]string) ([]acceptanceTest, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var tests []acceptanceTest + for _, e := range entries { + d := filepath.Join(dir, e.Name()) + name, err := os.ReadFile(filepath.Join(d, "name")) + if !e.IsDir() || errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + counters, err := scanDir(d, metas) + if err != nil { + return nil, err + } + if len(counters) > 0 { + tests = append(tests, acceptanceTest{name: string(name), counters: counters}) + } + } + slices.SortFunc(tests, func(a, b acceptanceTest) int { return strings.Compare(a.name, b.name) }) + return tests, nil +} + +func compareBlocks(a, b block) int { + return cmp.Or(cmp.Compare(a.sl, b.sl), cmp.Compare(a.sc, b.sc), cmp.Compare(a.el, b.el), cmp.Compare(a.ec, b.ec), cmp.Compare(a.stmts, b.stmts)) +} + +// catalog numbers every block of every package: the blocks of a file get consecutive ids, in +// the order of the file, and the files are in the order of their paths. +type catalog struct { + module string + layouts []*layout + layoutOf []int32 // block id -> layout + files map[string]int +} + +func newCatalog(pkgs map[[16]byte]*pkgMeta) (*catalog, error) { + byFile := map[string]map[block]struct{}{} + c := &catalog{files: map[string]int{}} + for _, p := range pkgs { + if c.module == "" { + c.module = p.module + } else if p.module != c.module { + return nil, fmt.Errorf("packages of modules %q and %q", c.module, p.module) + } + for _, units := range p.funcs { + for _, u := range units { + if byFile[u.file] == nil { + byFile[u.file] = map[block]struct{}{} + } + byFile[u.file][u] = struct{}{} + } + } + } + prefix := c.module + "/" + id := map[block]int32{} + for _, file := range slices.Sorted(maps.Keys(byFile)) { + path, ok := strings.CutPrefix(file, prefix) + if !ok { + return nil, fmt.Errorf("%s is outside module %s", file, c.module) + } + l := &layout{path: path, blocks: slices.SortedFunc(maps.Keys(byFile[file]), compareBlocks), first: len(c.layoutOf)} + for _, b := range l.blocks { + id[b] = int32(len(c.layoutOf)) + c.layoutOf = append(c.layoutOf, int32(len(c.layouts))) + } + c.files[file] = len(c.layouts) + c.layouts = append(c.layouts, l) + } + for _, p := range pkgs { + p.ids = make([][]int32, len(p.funcs)) + for f, units := range p.funcs { + p.ids[f] = make([]int32, len(units)) + for u, b := range units { + p.ids[f][u] = id[b] + } + } + } + return c, nil +} + +// inventory is the layouts of the files of the packages of the given meta-data files. +func (c *catalog) inventory(metas map[[16]byte]string, pkgsOf map[[16]byte][]*pkgMeta) []int { + set := map[int]struct{}{} + for h := range metas { + for _, p := range pkgsOf[h] { + for _, units := range p.funcs { + for _, u := range units { + set[c.files[u.file]] = struct{}{} + } + } + } + } + return slices.Sorted(maps.Keys(set)) +} + +// covered is the layouts that have a bit set. +func (c *catalog) covered(b bitset) []int { + var files []int + for w, word := range b { + for word != 0 { + l := int(c.layoutOf[w*64+bits.TrailingZeros64(word)]) + if len(files) == 0 || files[len(files)-1] != l { + files = append(files, l) + } + word &= word - 1 + } + } + return files +} + +// readSources fills in the git blob sha and the line count of each layout, and returns how +// many files were not found. +func readSources(root string, layouts []*layout) int { + missing := 0 + for _, l := range layouts { + data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(l.path))) + if err != nil { + missing++ + continue + } + h := sha1.New() + h.Write([]byte("blob " + strconv.Itoa(len(data)) + "\x00")) + h.Write(data) + copy(l.sha[:], h.Sum(nil)) + l.lines = strings.Count(string(data), "\n") + if len(data) > 0 && data[len(data)-1] != '\n' { + l.lines++ + } + } + return missing +} + +// parallel runs f(i) for i in [0, n) on all CPUs and returns the errors joined. +func parallel(n int, f func(i int) error) error { + errs := make([]error, n) + var next atomic.Int64 + var wg sync.WaitGroup + for range runtime.NumCPU() { + wg.Go(func() { + for i := int(next.Add(1)) - 1; i < n; i = int(next.Add(1)) - 1 { + errs[i] = f(i) + } + }) + } + wg.Wait() + return errors.Join(errs...) +} + +func pack(o options) error { + start := time.Now() + var meta map[string]any + if err := json.Unmarshal([]byte(o.meta), &meta); err != nil { + return fmt.Errorf("-meta: %w", err) + } + + unitMetas, accMetas := map[[16]byte]string{}, map[[16]byte]string{} + var unitCounters []string + if o.unitDir != "" { + var err error + unitCounters, err = scanDir(o.unitDir, unitMetas) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + var tests []acceptanceTest + if o.accDir != "" { + var err error + if tests, err = acceptanceTests(o.accDir, accMetas); err != nil { + return err + } + } + + cache := map[[16]byte]*pkgMeta{} + pkgsOf := map[[16]byte][]*pkgMeta{} + for _, metas := range []map[[16]byte]string{unitMetas, accMetas} { + for h, path := range metas { + if _, ok := pkgsOf[h]; ok { + continue + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if pkgsOf[h], err = readMeta(data, cache); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + } + cat, err := newCatalog(cache) + if err != nil { + return err + } + metaTime := time.Since(start) + + // One job per unit counter file (the unit tests write hundreds into one directory) and one + // per acceptance test. + jobs := make([][]string, 0, len(unitCounters)+len(tests)) + for _, f := range unitCounters { + jobs = append(jobs, []string{f}) + } + for _, t := range tests { + jobs = append(jobs, t.counters) + } + results := make([]bitset, len(jobs)) + err = parallel(len(jobs), func(i int) error { + results[i] = newBitset(len(cat.layoutOf)) + for _, path := range jobs[i] { + data, err := os.ReadFile(path) + if err != nil { + return err + } + var unknown error + err = readCounters(data, func(h [16]byte, pkg, fn uint32, counters []uint32) { + pkgs := pkgsOf[h] + if int(pkg) >= len(pkgs) || int(fn) >= len(pkgs[pkg].ids) { + unknown = fmt.Errorf("counters of function %d of package %d of meta-data %x, which it does not have", fn, pkg, h) + return + } + ids := pkgs[pkg].ids[fn] + for j, v := range counters { + if v != 0 && j < len(ids) { + results[i].set(ids[j]) + } + } + }) + if err = cmp.Or(err, unknown); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + return nil + }) + if err != nil { + return err + } + countersTime := time.Since(start) - metaTime + + var profiles []profile + accInventory, unitInventory := cat.inventory(accMetas, pkgsOf), cat.inventory(unitMetas, pkgsOf) + if len(tests) > 0 { + all := newBitset(len(cat.layoutOf)) + for _, b := range results[len(unitCounters):] { + all.or(b) + } + profiles = append(profiles, profile{kind: kindAll, files: accInventory, bits: all}) + } + if len(unitCounters) > 0 { + unit := newBitset(len(cat.layoutOf)) + for _, b := range results[:len(unitCounters)] { + unit.or(b) + } + profiles = append(profiles, profile{kind: kindUnit, files: unitInventory, bits: unit}) + } + for i, t := range tests { + b := results[len(unitCounters)+i] + profiles = append(profiles, profile{kind: kindTest, name: t.name, files: cat.covered(b), bits: b}) + } + + missing := readSources(o.root, cat.layouts) + meta["go"] = runtime.Version() + meta["module"] = cat.module + commit, err := exec.Command("git", "-C", o.root, "rev-parse", "HEAD").Output() + if err != nil { + return fmt.Errorf("git rev-parse HEAD in %s: %w", o.root, err) + } + meta["commit"] = strings.TrimSpace(string(commit)) + metaBytes, err := json.Marshal(meta) + if err != nil { + return err + } + data, distinct := encode(metaBytes, cat.layouts, profiles) + compressed, err := writeCompressed(o.output, data) + if err != nil { + return err + } + + blocksIn := func(inventory []int) int { + n := 0 + for _, l := range inventory { + n += len(cat.layouts[l].blocks) + } + return n + } + fmt.Fprintf(os.Stderr, "%s: %d acceptance tests, %d acceptance blocks, %d unit blocks, %d distinct file bitmaps, "+ + "%d bytes, %d compressed; %d meta-data files, %d counter files, %d sources not found; "+ + "meta-data %s, counters %s, total %s\n", + o.output, len(tests), blocksIn(accInventory), blocksIn(unitInventory), distinct, + len(data), compressed, len(pkgsOf), len(unitCounters)+countFiles(tests), missing, + metaTime.Round(time.Millisecond), countersTime.Round(time.Millisecond), time.Since(start).Round(time.Millisecond)) + return nil +} + +func countFiles(tests []acceptanceTest) int { + n := 0 + for _, t := range tests { + n += len(t.counters) + } + return n +} + +func writeCompressed(path string, data []byte) (int64, error) { + f, err := os.Create(path) + if err != nil { + return 0, err + } + w, err := zstd.NewWriter(f, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err == nil { + _, err = w.Write(data) + err = cmp.Or(err, w.Close()) + } + var size int64 + if fi, statErr := f.Stat(); statErr == nil { + size = fi.Size() + } + return size, cmp.Or(err, f.Close()) +} + +func dump(out io.Writer, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + data, err := io.ReadAll(r) + if err != nil { + return err + } + d, err := decode(data) + if err != nil { + return err + } + var meta struct { + Module string `json:"module"` + } + if err := json.Unmarshal(d.meta, &meta); err != nil { + return err + } + w := bufio.NewWriter(out) + if err := dumpText(w, d, meta.Module); err != nil { + return err + } + return w.Flush() +} diff --git a/tools/coverpack/main_test.go b/tools/coverpack/main_test.go new file mode 100644 index 00000000000..236c5f4de20 --- /dev/null +++ b/tools/coverpack/main_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const program = `package main + +import "os" + +func main() { + if len(os.Args) > 1 { + hello() + } else { + world() + } + for i := range 2 { + if i == 1 { os.Stdout.WriteString("one\n") } + } +} + +func hello() { os.Stdout.WriteString("hello\n") } + +func world() { + os.Stdout.WriteString("world\n") +} + +func never() int { + return 1 +} +` + +func run(t *testing.T, dir string, env []string, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), env...) + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) +} + +// textfmt is what `go tool covdata textfmt` reads from dirs: its profile lines, sorted. +func textfmt(t *testing.T, dirs ...string) []string { + out := filepath.Join(t.TempDir(), "profile.txt") + run(t, ".", nil, "go", "tool", "covdata", "textfmt", "-i", strings.Join(dirs, ","), "-o", out) + data, err := os.ReadFile(out) + require.NoError(t, err) + lines := strings.Split(strings.TrimSpace(string(data)), "\n")[1:] // mode: set + slices.Sort(lines) + return lines +} + +func covered(lines []string) []string { + return slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return strings.HasSuffix(l, " 0") }) +} + +// sections is the profile lines of each section of a dump, sorted. +func sections(dump string) map[string][]string { + result := map[string][]string{} + var name string + for line := range strings.Lines(dump) { + line = strings.TrimSuffix(line, "\n") + if h, ok := strings.CutPrefix(line, "# "); ok { + name = h + result[name] = []string{} + continue + } + result[name] = append(result[name], line) + } + for _, lines := range result { + slices.Sort(lines) + } + return result +} + +func TestPackMatchesCovdata(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "go.mod"), []byte("module example.com/m\n\ngo 1.22\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "main.go"), []byte(program), 0o644)) + git := []string{"GIT_CONFIG_GLOBAL=/dev/null", "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t"} + run(t, src, git, "git", "init", "-q") + run(t, src, git, "git", "add", ".") + run(t, src, git, "git", "commit", "-q", "-m", "program") + head, err := exec.Command("git", "-C", src, "rev-parse", "HEAD").Output() + require.NoError(t, err) + bin := filepath.Join(t.TempDir(), "bin") + run(t, src, []string{"GOWORK=off", "GOFLAGS="}, "go", "build", "-cover", "-o", bin, ".") + + acc, unit := t.TempDir(), t.TempDir() + for name, args := range map[string][]string{"TestAccept/hello": {"x"}, "TestAccept/world": nil} { + dir := filepath.Join(acc, filepath.Base(name)) + require.NoError(t, os.Mkdir(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "name"), []byte(name), 0o644)) + run(t, src, []string{"GOCOVERDIR=" + dir}, bin, args...) + } + run(t, src, []string{"GOCOVERDIR=" + unit}, bin) + // A directory without a name file is not a test, and one without counters ran nothing. + require.NoError(t, os.Mkdir(filepath.Join(acc, "other"), 0o755)) + + out := filepath.Join(t.TempDir(), "coverage.bin.zst") + require.NoError(t, pack(options{unitDir: unit, accDir: acc, root: src, meta: `{"cell": "test"}`, output: out})) + var buf bytes.Buffer + require.NoError(t, dump(&buf, out)) + got := sections(buf.String()) + + hello, world := filepath.Join(acc, "hello"), filepath.Join(acc, "world") + assert.Equal(t, textfmt(t, hello, world), got["all"]) + assert.Equal(t, textfmt(t, unit), got["unit"]) + assert.Equal(t, covered(textfmt(t, hello)), got["test TestAccept/hello"]) + assert.Equal(t, covered(textfmt(t, world)), got["test TestAccept/world"]) + assert.Len(t, got, 5) // coverage (the meta), all, unit and the two tests + assert.Contains(t, buf.String(), `"module":"example.com/m"`) + assert.Contains(t, buf.String(), `"commit":"`+strings.TrimSpace(string(head))+`"`) + assert.Contains(t, got["all"], "example.com/m/main.go:23.2,24.1 1 0") // never() did not run +} + +func TestDecodeRejectsTruncated(t *testing.T) { + data, _ := encode([]byte(`{}`), []*layout{{path: "a.go", blocks: []block{{file: "a.go", sl: 1, sc: 1, el: 2, ec: 2, stmts: 1}}}}, + []profile{{kind: kindTest, name: "T", files: []int{0}, bits: bitset{1}}}) + d, err := decode(data) + require.NoError(t, err) + assert.Equal(t, [][]byte{{1}}, d.bitmaps[0]) + for n := range len(data) { + _, err := decode(data[:n]) + assert.Error(t, err, "truncated to %d bytes", n) + } +} diff --git a/tools/go.mod b/tools/go.mod index ec2033f9088..baac5ac0407 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -5,6 +5,7 @@ go 1.27.0 toolchain go1.27.4 require ( + github.com/klauspost/compress v1.20.3 github.com/stretchr/testify v1.12.1 golang.org/x/mod v0.42.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/tools/go.sum b/tools/go.sum index 96cb2941ba9..6e05f35144d 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -394,6 +394,8 @@ github.com/kisielk/errcheck v1.20.0/go.mod h1:O+f80MKNwX8Oor2jwgpeQ9An7uJm+hRSgT github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=