102 lines
2.3 KiB
Go
102 lines
2.3 KiB
Go
package es7
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
elastic "github.com/elastic/go-elasticsearch/v7"
|
|
"github.com/elastic/go-elasticsearch/v7/esutil"
|
|
"github.com/loveuer/esgo2dump/model"
|
|
"github.com/loveuer/nf/nft/log"
|
|
)
|
|
|
|
func WriteData(ctx context.Context, client *elastic.Client, index string, docsCh <-chan []*model.ESSource, logs ...log.WroteLogger) error {
|
|
var (
|
|
err error
|
|
indexer esutil.BulkIndexer
|
|
total int
|
|
)
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case docs, ok := <-docsCh:
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
if len(docs) == 0 {
|
|
continue
|
|
}
|
|
|
|
count := 0
|
|
|
|
if indexer, err = esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
|
|
NumWorkers: 0,
|
|
FlushBytes: 0,
|
|
FlushInterval: 0,
|
|
Client: client,
|
|
Decoder: nil,
|
|
OnError: func(ctx context.Context, err error) {
|
|
log.Error("es7.writer: on error log, err = %s", err.Error())
|
|
},
|
|
Index: index,
|
|
ErrorTrace: true,
|
|
FilterPath: []string{},
|
|
Header: map[string][]string{},
|
|
Human: false,
|
|
Pipeline: "",
|
|
Pretty: false,
|
|
Refresh: "",
|
|
Routing: "",
|
|
Source: []string{},
|
|
SourceExcludes: []string{},
|
|
SourceIncludes: []string{},
|
|
Timeout: 0,
|
|
WaitForActiveShards: "",
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, doc := range docs {
|
|
var bs []byte
|
|
|
|
if bs, err = json.Marshal(doc.Content); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = indexer.Add(context.Background(), esutil.BulkIndexerItem{
|
|
Action: "index",
|
|
Index: index,
|
|
DocumentID: doc.DocId,
|
|
Body: bytes.NewReader(bs),
|
|
OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, item2 esutil.BulkIndexerResponseItem, bulkErr error) {
|
|
},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
count++
|
|
}
|
|
|
|
total += count
|
|
|
|
if err = indexer.Close(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
stats := indexer.Stats()
|
|
if stats.NumFailed > 0 {
|
|
return fmt.Errorf("write to es failed_count=%d bulk_count=%d", stats.NumFailed, count)
|
|
}
|
|
|
|
if len(logs) > 0 && logs[0] != nil {
|
|
logs[0].Info("Dump: succeed=%d total=%d docs succeed!!!", count, total)
|
|
}
|
|
}
|
|
}
|
|
}
|