loveuer/esgo2dump
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip: fix size = 0 error

Browse Source
This commit is contained in:
loveuer
2024-12-13 15:01:40 +08:00
parent dde92f2e59
commit 724c695eb7
3 changed files with 19 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
xes/es7/read.go
View File

@ -5,13 +5,14 @@ import (
"context"
"encoding/json"
"fmt"
"time"
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/loveuer/esgo2dump/internal/util"
"github.com/loveuer/esgo2dump/log"
"github.com/loveuer/esgo2dump/model"
"github.com/samber/lo"
"time"
)
// ReadData
@ -42,9 +43,7 @@ func ReadData(ctx context.Context, client *elastic.Client, index string, size, m
"scroll_id": scrollId,
})
var (
rr *esapi.Response
)
var rr *esapi.Response
if rr, err = client.ClearScroll(
client.ClearScroll.WithContext(util.Timeout(3)),
@ -184,7 +183,7 @@ func ReadDataV2(
sort = []string{}
}
if query != nil && len(query) > 0 {
if len(query) > 0 {
body["query"] = query
}
@ -200,10 +199,11 @@ func ReadDataV2(
}()
for {
ws := int(util.Min(size, max-total))
qs = []func(*esapi.SearchRequest){
client.Search.WithContext(util.TimeoutCtx(ctx, 30)),
client.Search.WithIndex(index),
client.Search.WithSize(int(util.Min(size, max-total))),
client.Search.WithSize(ws),
client.Search.WithSort(sorts...),
}
@ -221,6 +221,8 @@ func ReadDataV2(
return
}
log.Debug("es7.ReadDataV2: search request size = %d, body = %s", ws, string(bs))
qs = append(qs, client.Search.WithBody(bytes.NewReader(bs)))
if resp, err = client.Search(qs...); err != nil {
errCh <- err
@ -232,7 +234,7 @@ func ReadDataV2(
return
}
var result = new(model.ESResponseV7)
result := new(model.ESResponseV7)
decoder := json.NewDecoder(resp.Body)
if err = decoder.Decode(result); err != nil {
errCh <- err
@ -245,6 +247,7 @@ func ReadDataV2(
}
dataCh <- result.Hits.Hits
log.Debug("es7.ReadDataV2: search response hits = %d", len(result.Hits.Hits))
total += uint64(len(result.Hits.Hits))
if uint64(len(result.Hits.Hits)) < size || (max > 0 && total >= max) {
@ -253,9 +256,7 @@ func ReadDataV2(
searchAfter = result.Hits.Hits[len(result.Hits.Hits)-1].Sort
}
}()
return dataCh, errCh
}