introduce DisplayTruncator (#630) · boltless.me/zoekt@626c7d8

+4

BUILD.bazel

··· 30 30 "indexbuilder.go", 31 31 "indexdata.go", 32 32 "indexfile.go", 33 + "limit.go", 33 34 "marshal.go", 34 35 "matchiter.go", 35 36 "matchtree.go", ··· 110 111 "eval_test.go", 111 112 "hititer_test.go", 112 113 "index_test.go", 114 + "indexdata_test.go", 115 + "limit_test.go", 113 116 "marshal_test.go", 114 117 "matchtree_test.go", 115 118 "merge_test.go", ··· 127 130 "@com_github_grafana_regexp//:regexp", 128 131 "@com_github_roaringbitmap_roaring//:roaring", 129 132 "@org_golang_google_protobuf//proto", 133 + "@org_golang_x_exp//slices", 130 134 ], 131 135 )

+1 -99

eval.go

··· 388 388 389 389 // If document ranking is enabled, then we can rank and truncate the files to save memory. 390 390 if opts.UseDocumentRanks { 391 - truncateDocs := opts.MaxDocDisplayCount > 0 && opts.MaxDocDisplayCount < len(res.Files) 392 - truncateMatches := opts.MaxMatchDisplayCount > 0 393 - if truncateDocs || truncateMatches { 394 - SortFiles(res.Files) 395 - } 396 - if truncateDocs { 397 - res.Files = res.Files[:opts.MaxDocDisplayCount] 398 - } 399 - if truncateMatches { 400 - res.LimitMatches(opts.MaxMatchDisplayCount, opts.ChunkMatches) 401 - } 391 + res.Files = SortAndTruncateFiles(res.Files, opts) 402 392 } 403 393 404 394 return &res, nil ··· 830 820 } 831 821 return &bruteForceMatchTree{}, false, false, nil 832 822 } 833 - 834 - // Limits the number of matches on the SearchResult by the given positive limit, 835 - // returning the remaining limit, if any. 836 - func (result *SearchResult) LimitMatches(limit int, chunkMatches bool) int { 837 - var limiter func(file *FileMatch, limit int) int 838 - if chunkMatches { 839 - limiter = limitChunkMatches 840 - } else { 841 - limiter = limitLineMatches 842 - } 843 - for i := range result.Files { 844 - limit = limiter(&result.Files[i], limit) 845 - if limit == 0 { 846 - if i != len(result.Files)-1 { 847 - result.Files = result.Files[:i+1] 848 - } 849 - return 0 850 - } 851 - } 852 - return limit 853 - } 854 - 855 - // Limit the number of ChunkMatches in the given FileMatch, returning the 856 - // remaining limit, if any. 857 - func limitChunkMatches(file *FileMatch, limit int) int { 858 - for i := range file.ChunkMatches { 859 - cm := &file.ChunkMatches[i] 860 - if len(cm.Ranges) > limit { 861 - // We potentially need to effect the limit upon 3 different fields: 862 - // Ranges, SymbolInfo, and Content. 863 - 864 - // Content is the most complicated: we need to remove the last N 865 - // lines from it, where N is the difference between the line number 866 - // of the end of the old last Range and that of the new last Range. 867 - // This calculation is correct in the presence of both context lines 868 - // and multiline Ranges, taking into account that Content never has 869 - // a trailing newline. 870 - n := cm.Ranges[len(cm.Ranges)-1].End.LineNumber - cm.Ranges[limit-1].End.LineNumber 871 - if n > 0 { 872 - for b := len(cm.Content) - 1; b >= 0; b-- { 873 - if cm.Content[b] == '\n' { 874 - n -= 1 875 - } 876 - if n == 0 { 877 - cm.Content = cm.Content[:b] 878 - break 879 - } 880 - } 881 - if n > 0 { 882 - // Should be impossible. 883 - log.Panicf("Failed to find enough newlines when truncating Content, %d left over, %d ranges", n, len(cm.Ranges)) 884 - } 885 - } 886 - 887 - cm.Ranges = cm.Ranges[:limit] 888 - if cm.SymbolInfo != nil { 889 - // When non-nil, SymbolInfo is specified to have the same length 890 - // as Ranges. 891 - cm.SymbolInfo = cm.SymbolInfo[:limit] 892 - } 893 - } 894 - if len(cm.Ranges) == limit { 895 - file.ChunkMatches = file.ChunkMatches[:i+1] 896 - limit = 0 897 - break 898 - } 899 - limit -= len(cm.Ranges) 900 - } 901 - return limit 902 - } 903 - 904 - // Limit the number of LineMatches in the given FileMatch, returning the 905 - // remaining limit, if any. 906 - func limitLineMatches(file *FileMatch, limit int) int { 907 - for i := range file.LineMatches { 908 - lm := &file.LineMatches[i] 909 - if len(lm.LineFragments) > limit { 910 - lm.LineFragments = lm.LineFragments[:limit] 911 - } 912 - if len(lm.LineFragments) == limit { 913 - file.LineMatches = file.LineMatches[:i+1] 914 - limit = 0 915 - break 916 - } 917 - limit -= len(lm.LineFragments) 918 - } 919 - return limit 920 - }

-150

eval_test.go

··· 15 15 package zoekt 16 16 17 17 import ( 18 - "bytes" 19 18 "context" 20 - "fmt" 21 19 "hash/fnv" 22 20 "reflect" 23 21 "regexp/syntax" ··· 386 384 } 387 385 } 388 386 } 389 - 390 - func TestLimitMatches(t *testing.T) { 391 - cases := []struct { 392 - // Represents a SearchResult with three dimensions: 393 - // 1. outer slice is `Files` 394 - // 2. inner slice is `{Chunk,Line}Matches` 395 - // 3. value is the length of `Ranges`/`LineFragments` 396 - in [][]int 397 - limit int 398 - expected [][]int 399 - }{{ 400 - in: [][]int{{1, 1, 1}}, 401 - limit: 1, 402 - expected: [][]int{{1}}, 403 - }, { 404 - in: [][]int{{1, 1, 1}}, 405 - limit: 3, 406 - expected: [][]int{{1, 1, 1}}, 407 - }, { 408 - in: [][]int{{1, 1, 1}}, 409 - limit: 4, 410 - expected: [][]int{{1, 1, 1}}, 411 - }, { 412 - in: [][]int{{2, 2, 2}}, 413 - limit: 4, 414 - expected: [][]int{{2, 2}}, 415 - }, { 416 - in: [][]int{{2, 2, 2}}, 417 - limit: 3, 418 - expected: [][]int{{2, 1}}, 419 - }, { 420 - in: [][]int{{2, 2, 2}}, 421 - limit: 1, 422 - expected: [][]int{{1}}, 423 - }, { 424 - in: [][]int{{1}, {1}}, 425 - limit: 2, 426 - expected: [][]int{{1}, {1}}, 427 - }, { 428 - in: [][]int{{1}, {1}}, 429 - limit: 1, 430 - expected: [][]int{{1}}, 431 - }, { 432 - in: [][]int{{1}, {1, 3}}, 433 - limit: 4, 434 - expected: [][]int{{1}, {1, 2}}, 435 - }, { 436 - in: [][]int{{1}, {2, 2}, {3, 3, 3}}, 437 - limit: 4, 438 - expected: [][]int{{1}, {2, 1}}, 439 - }} 440 - 441 - for _, tc := range cases { 442 - t.Run("ChunkMatches", func(t *testing.T) { 443 - // Generate a ChunkMatch suitable for testing `LimitChunkMatches`. 444 - generateChunkMatch := func(numRanges, lineNumber int) (ChunkMatch, int) { 445 - cm := ChunkMatch{SymbolInfo: make([]*Symbol, numRanges)} 446 - 447 - // To simplify testing, we generate Content and the associated 448 - // Ranges with fixed logic: each ChunkMatch has 1 line of 449 - // context, and each Range spans two lines. It'd probably be 450 - // better to do some kind of property-based testing, but this is 451 - // alright. 452 - 453 - // 1 line of context. 454 - cm.Content = append(cm.Content, []byte("context\n")...) 455 - for i := 0; i < numRanges; i += 1 { 456 - cm.Ranges = append(cm.Ranges, Range{ 457 - // We only provide LineNumber as that's all that's 458 - // relevant. 459 - Start: Location{LineNumber: uint32(lineNumber + (2 * i) + 1)}, 460 - End: Location{LineNumber: uint32(lineNumber + (2 * i) + 2)}, 461 - }) 462 - cm.Content = append(cm.Content, []byte(fmt.Sprintf("range%dStart\nrange%dEnd\n", i, i))...) 463 - } 464 - // 1 line of context. Content in zoekt notably just does not 465 - // contain a trailing newline. 466 - cm.Content = append(cm.Content, []byte("context")...) 467 - 468 - // Next Chunk starts two lines past the number of lines we just 469 - // added. 470 - return cm, lineNumber + (2 * numRanges) + 4 471 - } 472 - 473 - res := SearchResult{} 474 - for _, file := range tc.in { 475 - fm := FileMatch{} 476 - lineNumber := 0 477 - for _, numRanges := range file { 478 - var cm ChunkMatch 479 - cm, lineNumber = generateChunkMatch(numRanges, lineNumber) 480 - fm.ChunkMatches = append(fm.ChunkMatches, cm) 481 - } 482 - res.Files = append(res.Files, fm) 483 - } 484 - 485 - res.LimitMatches(tc.limit, true) 486 - 487 - var got [][]int 488 - for _, fm := range res.Files { 489 - var matches []int 490 - for _, cm := range fm.ChunkMatches { 491 - if len(cm.Ranges) != len(cm.SymbolInfo) { 492 - t.Errorf("Expected Ranges and SymbolInfo to be the same size, but got %d and %d", len(cm.Ranges), len(cm.SymbolInfo)) 493 - } 494 - 495 - // Using the logic from generateChunkMatch. 496 - expectedNewlines := 1 + (len(cm.Ranges) * 2) 497 - actualNewlines := bytes.Count(cm.Content, []byte("\n")) 498 - if actualNewlines != expectedNewlines { 499 - t.Errorf("Expected Content to have %d newlines but got %d", expectedNewlines, actualNewlines) 500 - } 501 - 502 - matches = append(matches, len(cm.Ranges)) 503 - } 504 - got = append(got, matches) 505 - } 506 - if !cmp.Equal(tc.expected, got) { 507 - t.Errorf("Expected %v but got %v", tc.expected, got) 508 - } 509 - }) 510 - 511 - t.Run("LineMatches", func(t *testing.T) { 512 - res := SearchResult{} 513 - for _, file := range tc.in { 514 - fm := FileMatch{} 515 - for _, numFragments := range file { 516 - fm.LineMatches = append(fm.LineMatches, LineMatch{LineFragments: make([]LineFragmentMatch, numFragments)}) 517 - } 518 - res.Files = append(res.Files, fm) 519 - } 520 - 521 - res.LimitMatches(tc.limit, false) 522 - 523 - var got [][]int 524 - for _, fm := range res.Files { 525 - var matches []int 526 - for _, lm := range fm.LineMatches { 527 - matches = append(matches, len(lm.LineFragments)) 528 - } 529 - got = append(got, matches) 530 - } 531 - if !cmp.Equal(tc.expected, got) { 532 - t.Errorf("Expected %v but got %v", tc.expected, got) 533 - } 534 - }) 535 - } 536 - }

+1

internal/tracer/BUILD.bazel

··· 13 13 "//internal/otlpenv", 14 14 "@com_github_opentracing_opentracing_go//:opentracing-go", 15 15 "@com_github_pkg_errors//:errors", 16 + "@com_github_sourcegraph_log//:log", 16 17 "@com_github_uber_jaeger_client_go//:jaeger-client-go", 17 18 "@com_github_uber_jaeger_client_go//config", 18 19 "@com_github_uber_jaeger_lib//metrics",

+145

limit.go

··· 1 + package zoekt 2 + 3 + import "log" 4 + 5 + // SortAndTruncateFiles is a convenience around SortFiles and 6 + // DisplayTruncator. Given an aggregated files it will sort and then truncate 7 + // based on the search options. 8 + func SortAndTruncateFiles(files []FileMatch, opts *SearchOptions) []FileMatch { 9 + SortFiles(files) 10 + truncator, _ := NewDisplayTruncator(opts) 11 + files, _ = truncator(files) 12 + return files 13 + } 14 + 15 + // DisplayTruncator is a stateful function which enforces Document and Match 16 + // display limits by truncating and mutating before. hasMore is true until the 17 + // limits are exhausted. Once hasMore is false each subsequent call will 18 + // return an empty after and hasMore false. 19 + type DisplayTruncator func(before []FileMatch) (after []FileMatch, hasMore bool) 20 + 21 + // NewDisplayTruncator will return a DisplayTruncator which enforces the limits in 22 + // opts. If there are no limits to enforce, hasLimits is false and there is no 23 + // need to call DisplayTruncator. 24 + func NewDisplayTruncator(opts *SearchOptions) (_ DisplayTruncator, hasLimits bool) { 25 + docLimit := opts.MaxDocDisplayCount 26 + docLimited := docLimit > 0 27 + 28 + matchLimit := opts.MaxMatchDisplayCount 29 + matchLimited := matchLimit > 0 30 + 31 + done := false 32 + 33 + if !docLimited && !matchLimited { 34 + return func(fm []FileMatch) ([]FileMatch, bool) { 35 + return fm, true 36 + }, false 37 + } 38 + 39 + return func(fm []FileMatch) ([]FileMatch, bool) { 40 + if done { 41 + return nil, false 42 + } 43 + 44 + if docLimited { 45 + if len(fm) >= docLimit { 46 + done = true 47 + return fm[:docLimit], false 48 + } 49 + docLimit -= len(fm) 50 + } 51 + 52 + if matchLimited { 53 + fm, matchLimit = limitMatches(fm, matchLimit, opts.ChunkMatches) 54 + if matchLimit <= 0 { 55 + done = true 56 + return fm, false 57 + } 58 + } 59 + 60 + return fm, true 61 + }, true 62 + } 63 + 64 + func limitMatches(files []FileMatch, limit int, chunkMatches bool) ([]FileMatch, int) { 65 + var limiter func(file *FileMatch, limit int) int 66 + if chunkMatches { 67 + limiter = limitChunkMatches 68 + } else { 69 + limiter = limitLineMatches 70 + } 71 + for i := range files { 72 + limit = limiter(&files[i], limit) 73 + if limit <= 0 { 74 + return files[:i+1], 0 75 + } 76 + } 77 + return files, limit 78 + } 79 + 80 + // Limit the number of ChunkMatches in the given FileMatch, returning the 81 + // remaining limit, if any. 82 + func limitChunkMatches(file *FileMatch, limit int) int { 83 + for i := range file.ChunkMatches { 84 + cm := &file.ChunkMatches[i] 85 + if len(cm.Ranges) > limit { 86 + // We potentially need to effect the limit upon 3 different fields: 87 + // Ranges, SymbolInfo, and Content. 88 + 89 + // Content is the most complicated: we need to remove the last N 90 + // lines from it, where N is the difference between the line number 91 + // of the end of the old last Range and that of the new last Range. 92 + // This calculation is correct in the presence of both context lines 93 + // and multiline Ranges, taking into account that Content never has 94 + // a trailing newline. 95 + n := cm.Ranges[len(cm.Ranges)-1].End.LineNumber - cm.Ranges[limit-1].End.LineNumber 96 + if n > 0 { 97 + for b := len(cm.Content) - 1; b >= 0; b-- { 98 + if cm.Content[b] == '\n' { 99 + n -= 1 100 + } 101 + if n == 0 { 102 + cm.Content = cm.Content[:b] 103 + break 104 + } 105 + } 106 + if n > 0 { 107 + // Should be impossible. 108 + log.Panicf("Failed to find enough newlines when truncating Content, %d left over, %d ranges", n, len(cm.Ranges)) 109 + } 110 + } 111 + 112 + cm.Ranges = cm.Ranges[:limit] 113 + if cm.SymbolInfo != nil { 114 + // When non-nil, SymbolInfo is specified to have the same length 115 + // as Ranges. 116 + cm.SymbolInfo = cm.SymbolInfo[:limit] 117 + } 118 + } 119 + if len(cm.Ranges) == limit { 120 + file.ChunkMatches = file.ChunkMatches[:i+1] 121 + limit = 0 122 + break 123 + } 124 + limit -= len(cm.Ranges) 125 + } 126 + return limit 127 + } 128 + 129 + // Limit the number of LineMatches in the given FileMatch, returning the 130 + // remaining limit, if any. 131 + func limitLineMatches(file *FileMatch, limit int) int { 132 + for i := range file.LineMatches { 133 + lm := &file.LineMatches[i] 134 + if len(lm.LineFragments) > limit { 135 + lm.LineFragments = lm.LineFragments[:limit] 136 + } 137 + if len(lm.LineFragments) == limit { 138 + file.LineMatches = file.LineMatches[:i+1] 139 + limit = 0 140 + break 141 + } 142 + limit -= len(lm.LineFragments) 143 + } 144 + return limit 145 + }

+163

limit_test.go

··· 1 + package zoekt 2 + 3 + import ( 4 + "bytes" 5 + "fmt" 6 + "testing" 7 + 8 + "github.com/google/go-cmp/cmp" 9 + ) 10 + 11 + func TestLimitMatches(t *testing.T) { 12 + cases := []struct { 13 + // Represents a SearchResult with three dimensions: 14 + // 1. outer slice is `Files` 15 + // 2. inner slice is `{Chunk,Line}Matches` 16 + // 3. value is the length of `Ranges`/`LineFragments` 17 + in [][]int 18 + limit int 19 + expected [][]int 20 + }{{ 21 + in: [][]int{{1, 1, 1}}, 22 + limit: 1, 23 + expected: [][]int{{1}}, 24 + }, { 25 + in: [][]int{{1, 1, 1}}, 26 + limit: 3, 27 + expected: [][]int{{1, 1, 1}}, 28 + }, { 29 + in: [][]int{{1, 1, 1}}, 30 + limit: 4, 31 + expected: [][]int{{1, 1, 1}}, 32 + }, { 33 + in: [][]int{{2, 2, 2}}, 34 + limit: 4, 35 + expected: [][]int{{2, 2}}, 36 + }, { 37 + in: [][]int{{2, 2, 2}}, 38 + limit: 3, 39 + expected: [][]int{{2, 1}}, 40 + }, { 41 + in: [][]int{{2, 2, 2}}, 42 + limit: 1, 43 + expected: [][]int{{1}}, 44 + }, { 45 + in: [][]int{{1}, {1}}, 46 + limit: 2, 47 + expected: [][]int{{1}, {1}}, 48 + }, { 49 + in: [][]int{{1}, {1}}, 50 + limit: 1, 51 + expected: [][]int{{1}}, 52 + }, { 53 + in: [][]int{{1}, {1, 3}}, 54 + limit: 4, 55 + expected: [][]int{{1}, {1, 2}}, 56 + }, { 57 + in: [][]int{{1}, {2, 2}, {3, 3, 3}}, 58 + limit: 4, 59 + expected: [][]int{{1}, {2, 1}}, 60 + }} 61 + 62 + for _, tc := range cases { 63 + t.Run("ChunkMatches", func(t *testing.T) { 64 + // Generate a ChunkMatch suitable for testing `LimitChunkMatches`. 65 + generateChunkMatch := func(numRanges, lineNumber int) (ChunkMatch, int) { 66 + cm := ChunkMatch{SymbolInfo: make([]*Symbol, numRanges)} 67 + 68 + // To simplify testing, we generate Content and the associated 69 + // Ranges with fixed logic: each ChunkMatch has 1 line of 70 + // context, and each Range spans two lines. It'd probably be 71 + // better to do some kind of property-based testing, but this is 72 + // alright. 73 + 74 + // 1 line of context. 75 + cm.Content = append(cm.Content, []byte("context\n")...) 76 + for i := 0; i < numRanges; i += 1 { 77 + cm.Ranges = append(cm.Ranges, Range{ 78 + // We only provide LineNumber as that's all that's 79 + // relevant. 80 + Start: Location{LineNumber: uint32(lineNumber + (2 * i) + 1)}, 81 + End: Location{LineNumber: uint32(lineNumber + (2 * i) + 2)}, 82 + }) 83 + cm.Content = append(cm.Content, []byte(fmt.Sprintf("range%dStart\nrange%dEnd\n", i, i))...) 84 + } 85 + // 1 line of context. Content in zoekt notably just does not 86 + // contain a trailing newline. 87 + cm.Content = append(cm.Content, []byte("context")...) 88 + 89 + // Next Chunk starts two lines past the number of lines we just 90 + // added. 91 + return cm, lineNumber + (2 * numRanges) + 4 92 + } 93 + 94 + res := SearchResult{} 95 + for _, file := range tc.in { 96 + fm := FileMatch{} 97 + lineNumber := 0 98 + for _, numRanges := range file { 99 + var cm ChunkMatch 100 + cm, lineNumber = generateChunkMatch(numRanges, lineNumber) 101 + fm.ChunkMatches = append(fm.ChunkMatches, cm) 102 + } 103 + res.Files = append(res.Files, fm) 104 + } 105 + 106 + res.Files = SortAndTruncateFiles(res.Files, &SearchOptions{ 107 + MaxMatchDisplayCount: tc.limit, 108 + ChunkMatches: true, 109 + }) 110 + 111 + var got [][]int 112 + for _, fm := range res.Files { 113 + var matches []int 114 + for _, cm := range fm.ChunkMatches { 115 + if len(cm.Ranges) != len(cm.SymbolInfo) { 116 + t.Errorf("Expected Ranges and SymbolInfo to be the same size, but got %d and %d", len(cm.Ranges), len(cm.SymbolInfo)) 117 + } 118 + 119 + // Using the logic from generateChunkMatch. 120 + expectedNewlines := 1 + (len(cm.Ranges) * 2) 121 + actualNewlines := bytes.Count(cm.Content, []byte("\n")) 122 + if actualNewlines != expectedNewlines { 123 + t.Errorf("Expected Content to have %d newlines but got %d", expectedNewlines, actualNewlines) 124 + } 125 + 126 + matches = append(matches, len(cm.Ranges)) 127 + } 128 + got = append(got, matches) 129 + } 130 + if !cmp.Equal(tc.expected, got) { 131 + t.Errorf("Expected %v but got %v", tc.expected, got) 132 + } 133 + }) 134 + 135 + t.Run("LineMatches", func(t *testing.T) { 136 + res := SearchResult{} 137 + for _, file := range tc.in { 138 + fm := FileMatch{} 139 + for _, numFragments := range file { 140 + fm.LineMatches = append(fm.LineMatches, LineMatch{LineFragments: make([]LineFragmentMatch, numFragments)}) 141 + } 142 + res.Files = append(res.Files, fm) 143 + } 144 + 145 + res.Files = SortAndTruncateFiles(res.Files, &SearchOptions{ 146 + MaxMatchDisplayCount: tc.limit, 147 + ChunkMatches: false, 148 + }) 149 + 150 + var got [][]int 151 + for _, fm := range res.Files { 152 + var matches []int 153 + for _, lm := range fm.LineMatches { 154 + matches = append(matches, len(lm.LineFragments)) 155 + } 156 + got = append(got, matches) 157 + } 158 + if !cmp.Equal(tc.expected, got) { 159 + t.Errorf("Expected %v but got %v", tc.expected, got) 160 + } 161 + }) 162 + } 163 + }

+8 -29

shards/aggregate.go

··· 49 49 if len(r.Files) > 0 { 50 50 c.aggregate.Files = append(c.aggregate.Files, r.Files...) 51 51 52 - zoekt.SortFiles(c.aggregate.Files) 53 - if max := c.opts.MaxDocDisplayCount; max > 0 && max < len(c.aggregate.Files) { 54 - c.aggregate.Files = c.aggregate.Files[:max] 55 - } 56 - if max := c.opts.MaxMatchDisplayCount; max > 0 { 57 - c.aggregate.LimitMatches(max, c.opts.ChunkMatches) 58 - } 52 + c.aggregate.Files = zoekt.SortAndTruncateFiles(c.aggregate.Files, c.opts) 59 53 60 54 for k, v := range r.RepoURLs { 61 55 c.aggregate.RepoURLs[k] = v ··· 157 151 }), finalFlush 158 152 } 159 153 160 - // limitSender wraps a sender and calls cancel once it has seen either of 161 - // `docLimit` files or `matchLimit` matches. 162 - func limitSender(cancel context.CancelFunc, sender zoekt.Sender, docLimit, matchLimit int, chunkMatches bool) zoekt.Sender { 163 - docLimited := docLimit > 0 164 - matchLimited := matchLimit > 0 154 + // limitSender wraps a sender and calls cancel once the truncator has finished 155 + // truncating. 156 + func limitSender(cancel context.CancelFunc, sender zoekt.Sender, truncator zoekt.DisplayTruncator) zoekt.Sender { 165 157 return stream.SenderFunc(func(result *zoekt.SearchResult) { 166 - // We cancel when we hit either of the two limits. It is assumed that, 167 - // after canceling, we are not called again. 168 - 169 - if docLimited { 170 - if len(result.Files) >= docLimit { 171 - result.Files = result.Files[:docLimit] 172 - cancel() 173 - } 174 - docLimit -= len(result.Files) 158 + var hasMore bool 159 + result.Files, hasMore = truncator(result.Files) 160 + if !hasMore { 161 + cancel() 175 162 } 176 - 177 - if matchLimited { 178 - matchLimit = result.LimitMatches(matchLimit, chunkMatches) 179 - if matchLimit == 0 { 180 - cancel() 181 - } 182 - } 183 - 184 163 sender.Send(result) 185 164 }) 186 165 }

+2 -2

shards/shards.go

··· 585 585 // For streaming, the wrapping has to happen in the inverted order. 586 586 sender = copyFileSender(sender) 587 587 588 - if opts.MaxDocDisplayCount > 0 || opts.MaxMatchDisplayCount > 0 { 588 + if truncator, hasLimits := zoekt.NewDisplayTruncator(opts); hasLimits { 589 589 var cancel context.CancelFunc 590 590 ctx, cancel = context.WithCancel(ctx) 591 591 defer cancel() 592 - sender = limitSender(cancel, sender, opts.MaxDocDisplayCount, opts.MaxMatchDisplayCount, opts.ChunkMatches) 592 + sender = limitSender(cancel, sender, truncator) 593 593 } 594 594 595 595 sender, flush := newFlushCollectSender(opts, sender)

Configure Feed

Configure Feed