loveuer/esgo2dump
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: when max not set, dump size negative

Browse Source
This commit is contained in:
loveuer
2024-07-22 10:13:12 +08:00
parent beb2ca4cf4
commit ebb4365135
8 changed files with 27 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
xes/es6/read.go
View File

@ -14,7 +14,7 @@ import (
"time"
)
func ReadData(ctx context.Context, client *elastic.Client, index string, size, max int, query map[string]any, source []string, sort []string) (<-chan []*model.ESSource, <-chan error) {
func ReadData(ctx context.Context, client *elastic.Client, index string, size, max uint64, query map[string]any, source []string, sort []string) (<-chan []*model.ESSource, <-chan error) {
var (
dataCh = make(chan []*model.ESSource)
errCh = make(chan error)
@ -26,7 +26,7 @@ func ReadData(ctx context.Context, client *elastic.Client, index string, size, m
resp *esapi.Response
result = new(model.ESResponseV6)
scrollId string
total int
total uint64
)
defer func() {
@ -63,7 +63,7 @@ func ReadData(ctx context.Context, client *elastic.Client, index string, size, m
qs := []func(*esapi.SearchRequest){
client.Search.WithContext(util.TimeoutCtx(ctx, 20)),
client.Search.WithIndex(index),
client.Search.WithSize(size),
client.Search.WithSize(int(size)),
client.Search.WithFrom(0),
client.Search.WithScroll(time.Duration(120) * time.Second),
}
@ -106,9 +106,9 @@ func ReadData(ctx context.Context, client *elastic.Client, index string, size, m
scrollId = result.ScrollId
dataCh <- result.Hits.Hits
total += len(result.Hits.Hits)
total += uint64(len(result.Hits.Hits))
if len(result.Hits.Hits) < size || (max > 0 && total >= max) {
if uint64(len(result.Hits.Hits)) < size || (max > 0 && total >= max) {
return
}
@ -135,9 +135,9 @@ func ReadData(ctx context.Context, client *elastic.Client, index string, size, m
}
dataCh <- result.Hits.Hits
total += len(result.Hits.Hits)
total += uint64(len(result.Hits.Hits))
if len(result.Hits.Hits) < size || (max > 0 && total >= max) {
if uint64(len(result.Hits.Hits)) < size || (max > 0 && total >= max) {
break
}
}

14
xes/es7/read.go
View File

@ -159,7 +159,7 @@ func ReadDataV2(
ctx context.Context,
client *elastic.Client,
index string,
size, max int,
size, max uint64,
query map[string]any,
source []string,
sort []string,
@ -174,9 +174,9 @@ func ReadDataV2(
err error
bs []byte
resp *esapi.Response
searchAfter = make([]any, 0)
total = 0
body = make(map[string]any)
searchAfter = make([]any, 0)
total uint64 = 0
body = make(map[string]any)
qs []func(request *esapi.SearchRequest)
)
@ -203,7 +203,7 @@ func ReadDataV2(
qs = []func(*esapi.SearchRequest){
client.Search.WithContext(util.TimeoutCtx(ctx, 30)),
client.Search.WithIndex(index),
client.Search.WithSize(util.Min(size, max-total)),
client.Search.WithSize(int(util.Min(size, max-total))),
client.Search.WithSort(sorts...),
}
@ -245,9 +245,9 @@ func ReadDataV2(
}
dataCh <- result.Hits.Hits
total += len(result.Hits.Hits)
total += uint64(len(result.Hits.Hits))
if len(result.Hits.Hits) < size || (max > 0 && total >= max) {
if uint64(len(result.Hits.Hits)) < size || (max > 0 && total >= max) {
break
}