2024-03-22 18:05:47 +08:00
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2024-03-27 17:44:01 +08:00
|
|
|
"bufio"
|
2024-03-22 18:05:47 +08:00
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"net/url"
|
|
|
|
"os"
|
2024-05-08 19:02:49 +08:00
|
|
|
"strings"
|
2024-03-26 17:23:10 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
elastic "github.com/elastic/go-elasticsearch/v7"
|
|
|
|
"github.com/go-resty/resty/v2"
|
2024-03-26 17:23:10 +08:00
|
|
|
"github.com/loveuer/esgo2dump/internal/opt"
|
2025-02-05 18:07:53 +08:00
|
|
|
"github.com/loveuer/esgo2dump/internal/tool"
|
2024-03-26 17:23:10 +08:00
|
|
|
"github.com/loveuer/esgo2dump/internal/xfile"
|
2025-02-05 18:07:53 +08:00
|
|
|
"github.com/loveuer/esgo2dump/pkg/log"
|
|
|
|
"github.com/loveuer/esgo2dump/pkg/model"
|
|
|
|
"github.com/loveuer/esgo2dump/xes/es7"
|
2024-05-08 19:02:49 +08:00
|
|
|
"github.com/samber/lo"
|
2024-03-26 17:23:10 +08:00
|
|
|
"github.com/spf13/cobra"
|
2024-03-22 18:05:47 +08:00
|
|
|
)
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
func newIO(ctx context.Context, uri string, ioType model.IOType) (model.IO[map[string]any], error) {
|
|
|
|
type Version struct {
|
|
|
|
Name string
|
|
|
|
Version struct {
|
|
|
|
Number string `json:"number"`
|
|
|
|
} `json:"version"`
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
2024-03-22 18:05:47 +08:00
|
|
|
var (
|
2025-02-05 18:07:53 +08:00
|
|
|
err error
|
|
|
|
target *url.URL
|
|
|
|
rr *resty.Response
|
|
|
|
v Version
|
2024-03-22 18:05:47 +08:00
|
|
|
)
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if target, err = url.Parse(uri); err != nil {
|
|
|
|
log.Debug("parse uri failed, type = %s, uri = %s, err = %s", ioType, uri, err.Error())
|
|
|
|
return xfile.NewClient(uri, ioType)
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if err = tool.ValidScheme(target.Scheme); err != nil {
|
|
|
|
log.Debug("uri scheme check failed, type = %s, uri = %s", ioType, uri)
|
|
|
|
return xfile.NewClient(uri, ioType)
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
// elastic uri
|
|
|
|
index := strings.TrimPrefix(target.Path, "/")
|
|
|
|
if index == "" {
|
|
|
|
return nil, fmt.Errorf("uri invalid without index(path)")
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
log.Debug("%s uri es index = %s", ioType, index)
|
2024-03-22 18:05:47 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
versionURL := fmt.Sprintf("%s://%s", target.Scheme, strings.Split(target.Host, ",")[0])
|
|
|
|
log.Debug("%s version url = %s", ioType, versionURL)
|
|
|
|
if rr, err = opt.HttpClient.R().Get(versionURL); err != nil {
|
|
|
|
log.Debug("get uri es version failed, type = %s, uri = %s, version_url = %s, err = %s", ioType, uri, versionURL, err.Error())
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if err = json.Unmarshal(rr.Body(), &v); err != nil {
|
|
|
|
log.Debug("decode uri es version failed, type = %s, uri = %s, version_url = %s, err = %s", ioType, uri, versionURL, err.Error())
|
|
|
|
return nil, err
|
2024-05-08 19:02:49 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
log.Debug("%s uri es version = %s", ioType, v.Version.Number)
|
2024-03-26 17:31:38 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
mainVersion := strings.Split(v.Version.Number, ".")[0]
|
|
|
|
switch mainVersion {
|
|
|
|
case "8":
|
|
|
|
case "7":
|
|
|
|
var client *elastic.Client
|
|
|
|
if client, err = es7.NewClient(ctx, uri); err != nil {
|
|
|
|
return nil, err
|
2024-03-26 17:31:38 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
return es7.NewStreamer(ctx, client, index)
|
|
|
|
case "6":
|
2024-03-22 18:05:47 +08:00
|
|
|
default:
|
2025-02-05 18:07:53 +08:00
|
|
|
return nil, fmt.Errorf("es version not supported yet: %s", mainVersion)
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
2025-02-05 18:07:53 +08:00
|
|
|
|
|
|
|
return nil, nil
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
func run(cmd *cobra.Command, args []string) error {
|
2024-03-22 18:05:47 +08:00
|
|
|
var (
|
2025-02-05 18:07:53 +08:00
|
|
|
err error
|
|
|
|
input model.IO[map[string]any]
|
|
|
|
output model.IO[map[string]any]
|
2024-03-26 21:05:37 +08:00
|
|
|
)
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if input, err = newIO(cmd.Context(), opt.Cfg.Args.Input, model.Input); err != nil {
|
|
|
|
return err
|
2024-05-08 19:02:49 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if output, err = newIO(cmd.Context(), opt.Cfg.Args.Output, model.Output); err != nil {
|
|
|
|
return err
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
go func() {
|
|
|
|
<-cmd.Context().Done()
|
|
|
|
os.Exit(1)
|
|
|
|
}()
|
2024-03-27 17:44:01 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if opt.Cfg.Args.QueryFile != "" {
|
|
|
|
// query file
|
|
|
|
var (
|
|
|
|
items []map[string]any
|
|
|
|
qf *os.File
|
|
|
|
// wrote count
|
|
|
|
wc int
|
|
|
|
)
|
2024-12-13 15:01:40 +08:00
|
|
|
if qf, err = os.Open(opt.Cfg.Args.QueryFile); err != nil {
|
2025-02-05 18:07:53 +08:00
|
|
|
return err
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
scanner := bufio.NewScanner(qf)
|
2025-02-05 18:07:53 +08:00
|
|
|
// query count
|
|
|
|
qc := 0
|
2024-03-27 17:44:01 +08:00
|
|
|
for scanner.Scan() {
|
2025-02-05 18:07:53 +08:00
|
|
|
qc++
|
|
|
|
qm := make(map[string]any)
|
|
|
|
if err = json.Unmarshal(scanner.Bytes(), &qm); err != nil {
|
|
|
|
return err
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
for {
|
|
|
|
if items, err = input.ReadData(
|
|
|
|
opt.Cfg.Args.Limit,
|
|
|
|
qm,
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2024-03-27 17:44:01 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if len(items) == 0 {
|
|
|
|
break
|
|
|
|
}
|
2024-03-27 17:44:01 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc, err = output.WriteData(items); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2024-03-27 17:44:01 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc != len(items) {
|
|
|
|
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
|
|
|
|
}
|
2024-03-27 17:44:01 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
log.Info("Dump: query_file[%06d] dump success = %d", qc, wc)
|
|
|
|
}
|
|
|
|
}
|
2024-03-27 17:44:01 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if opt.Cfg.Args.Query != "" {
|
|
|
|
var (
|
|
|
|
items []map[string]any
|
|
|
|
qm = make(map[string]any)
|
|
|
|
wc int
|
|
|
|
)
|
2024-03-22 18:05:47 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if err = json.Unmarshal([]byte(opt.Cfg.Args.Query), &qm); err != nil {
|
|
|
|
return err
|
2024-05-24 17:27:52 +08:00
|
|
|
}
|
2024-06-21 17:09:06 +08:00
|
|
|
|
2024-05-20 16:11:50 +08:00
|
|
|
for {
|
2025-02-05 18:07:53 +08:00
|
|
|
if items, err = input.ReadData(
|
|
|
|
opt.Cfg.Args.Limit,
|
|
|
|
qm,
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
); err != nil {
|
2024-05-24 17:27:52 +08:00
|
|
|
return err
|
2025-02-05 18:07:53 +08:00
|
|
|
}
|
2025-01-22 17:29:12 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if len(items) == 0 {
|
|
|
|
break
|
2024-05-20 16:11:50 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc, err = output.WriteData(items); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2024-06-21 17:09:06 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc != len(items) {
|
|
|
|
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
|
|
|
|
}
|
2024-06-21 17:09:06 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
log.Info("Dump: query dump success = %d", wc)
|
|
|
|
}
|
|
|
|
}
|
2024-03-22 18:05:47 +08:00
|
|
|
|
|
|
|
var (
|
2025-02-05 18:07:53 +08:00
|
|
|
items []map[string]any
|
|
|
|
wc int
|
2024-03-22 18:05:47 +08:00
|
|
|
)
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
for {
|
|
|
|
if items, err = input.ReadData(
|
|
|
|
opt.Cfg.Args.Limit,
|
|
|
|
nil,
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
|
|
|
|
); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2024-03-22 18:05:47 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if len(items) == 0 {
|
|
|
|
break
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc, err = output.WriteData(items); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2024-03-22 18:05:47 +08:00
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
if wc != len(items) {
|
|
|
|
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
log.Info("Dump: query dump success = %d", wc)
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|
|
|
|
|
2025-02-05 18:07:53 +08:00
|
|
|
return nil
|
2024-03-22 18:05:47 +08:00
|
|
|
}
|