226 lines
5.1 KiB
Go
Raw Normal View History

2024-03-22 18:05:47 +08:00
package cmd
import (
"bufio"
2024-03-22 18:05:47 +08:00
"context"
"encoding/json"
"fmt"
"net/url"
"os"
2024-05-08 19:02:49 +08:00
"strings"
2024-03-26 17:23:10 +08:00
2025-02-05 18:07:53 +08:00
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/go-resty/resty/v2"
2024-03-26 17:23:10 +08:00
"github.com/loveuer/esgo2dump/internal/opt"
2025-02-05 18:07:53 +08:00
"github.com/loveuer/esgo2dump/internal/tool"
2024-03-26 17:23:10 +08:00
"github.com/loveuer/esgo2dump/internal/xfile"
2025-02-05 18:07:53 +08:00
"github.com/loveuer/esgo2dump/pkg/log"
"github.com/loveuer/esgo2dump/pkg/model"
"github.com/loveuer/esgo2dump/xes/es7"
2024-05-08 19:02:49 +08:00
"github.com/samber/lo"
2024-03-26 17:23:10 +08:00
"github.com/spf13/cobra"
2024-03-22 18:05:47 +08:00
)
2025-02-05 18:07:53 +08:00
func newIO(ctx context.Context, uri string, ioType model.IOType) (model.IO[map[string]any], error) {
type Version struct {
Name string
Version struct {
Number string `json:"number"`
} `json:"version"`
}
2024-03-22 18:05:47 +08:00
var (
2025-02-05 18:07:53 +08:00
err error
target *url.URL
rr *resty.Response
v Version
2024-03-22 18:05:47 +08:00
)
2025-02-05 18:07:53 +08:00
if target, err = url.Parse(uri); err != nil {
log.Debug("parse uri failed, type = %s, uri = %s, err = %s", ioType, uri, err.Error())
return xfile.NewClient(uri, ioType)
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
if err = tool.ValidScheme(target.Scheme); err != nil {
log.Debug("uri scheme check failed, type = %s, uri = %s", ioType, uri)
return xfile.NewClient(uri, ioType)
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
// elastic uri
index := strings.TrimPrefix(target.Path, "/")
if index == "" {
return nil, fmt.Errorf("uri invalid without index(path)")
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
log.Debug("%s uri es index = %s", ioType, index)
2024-03-22 18:05:47 +08:00
2025-02-05 18:07:53 +08:00
versionURL := fmt.Sprintf("%s://%s", target.Scheme, strings.Split(target.Host, ",")[0])
log.Debug("%s version url = %s", ioType, versionURL)
if rr, err = opt.HttpClient.R().Get(versionURL); err != nil {
log.Debug("get uri es version failed, type = %s, uri = %s, version_url = %s, err = %s", ioType, uri, versionURL, err.Error())
}
2025-02-05 18:07:53 +08:00
if err = json.Unmarshal(rr.Body(), &v); err != nil {
log.Debug("decode uri es version failed, type = %s, uri = %s, version_url = %s, err = %s", ioType, uri, versionURL, err.Error())
return nil, err
2024-05-08 19:02:49 +08:00
}
2025-02-05 18:07:53 +08:00
log.Debug("%s uri es version = %s", ioType, v.Version.Number)
2025-02-05 18:07:53 +08:00
mainVersion := strings.Split(v.Version.Number, ".")[0]
switch mainVersion {
case "8":
case "7":
var client *elastic.Client
if client, err = es7.NewClient(ctx, uri); err != nil {
return nil, err
}
2025-02-05 18:07:53 +08:00
return es7.NewStreamer(ctx, client, index)
case "6":
2024-03-22 18:05:47 +08:00
default:
2025-02-05 18:07:53 +08:00
return nil, fmt.Errorf("es version not supported yet: %s", mainVersion)
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
return nil, nil
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
func run(cmd *cobra.Command, args []string) error {
2024-03-22 18:05:47 +08:00
var (
2025-02-05 18:07:53 +08:00
err error
input model.IO[map[string]any]
output model.IO[map[string]any]
2024-03-26 21:05:37 +08:00
)
2025-02-05 18:07:53 +08:00
if input, err = newIO(cmd.Context(), opt.Cfg.Args.Input, model.Input); err != nil {
return err
2024-05-08 19:02:49 +08:00
}
2025-02-05 18:07:53 +08:00
if output, err = newIO(cmd.Context(), opt.Cfg.Args.Output, model.Output); err != nil {
return err
}
2025-02-05 18:07:53 +08:00
go func() {
<-cmd.Context().Done()
os.Exit(1)
}()
2025-02-05 18:07:53 +08:00
if opt.Cfg.Args.QueryFile != "" {
// query file
var (
items []map[string]any
qf *os.File
// wrote count
wc int
)
if qf, err = os.Open(opt.Cfg.Args.QueryFile); err != nil {
2025-02-05 18:07:53 +08:00
return err
}
scanner := bufio.NewScanner(qf)
2025-02-05 18:07:53 +08:00
// query count
qc := 0
for scanner.Scan() {
2025-02-05 18:07:53 +08:00
qc++
qm := make(map[string]any)
if err = json.Unmarshal(scanner.Bytes(), &qm); err != nil {
return err
}
2025-02-05 18:07:53 +08:00
for {
if items, err = input.ReadData(
opt.Cfg.Args.Limit,
qm,
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
); err != nil {
return err
}
2025-02-05 18:07:53 +08:00
if len(items) == 0 {
break
}
2025-02-05 18:07:53 +08:00
if wc, err = output.WriteData(items); err != nil {
return err
}
2025-02-05 18:07:53 +08:00
if wc != len(items) {
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
}
2025-02-05 18:07:53 +08:00
log.Info("Dump: query_file[%06d] dump success = %d", qc, wc)
}
}
}
2025-02-05 18:07:53 +08:00
if opt.Cfg.Args.Query != "" {
var (
items []map[string]any
qm = make(map[string]any)
wc int
)
2024-03-22 18:05:47 +08:00
2025-02-05 18:07:53 +08:00
if err = json.Unmarshal([]byte(opt.Cfg.Args.Query), &qm); err != nil {
return err
2024-05-24 17:27:52 +08:00
}
2024-06-21 17:09:06 +08:00
for {
2025-02-05 18:07:53 +08:00
if items, err = input.ReadData(
opt.Cfg.Args.Limit,
qm,
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
); err != nil {
2024-05-24 17:27:52 +08:00
return err
2025-02-05 18:07:53 +08:00
}
2025-01-22 17:29:12 +08:00
2025-02-05 18:07:53 +08:00
if len(items) == 0 {
break
}
2025-02-05 18:07:53 +08:00
if wc, err = output.WriteData(items); err != nil {
return err
}
2024-06-21 17:09:06 +08:00
2025-02-05 18:07:53 +08:00
if wc != len(items) {
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
}
2024-06-21 17:09:06 +08:00
2025-02-05 18:07:53 +08:00
log.Info("Dump: query dump success = %d", wc)
}
}
2024-03-22 18:05:47 +08:00
var (
2025-02-05 18:07:53 +08:00
items []map[string]any
wc int
2024-03-22 18:05:47 +08:00
)
2025-02-05 18:07:53 +08:00
for {
if items, err = input.ReadData(
opt.Cfg.Args.Limit,
nil,
lo.Filter(strings.Split(opt.Cfg.Args.Field, ","), func(x string, _ int) bool { return x != "" }),
lo.Filter(strings.Split(opt.Cfg.Args.Sort, ","), func(x string, _ int) bool { return x != "" }),
); err != nil {
return err
}
2024-03-22 18:05:47 +08:00
2025-02-05 18:07:53 +08:00
if len(items) == 0 {
break
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
if wc, err = output.WriteData(items); err != nil {
return err
}
2024-03-22 18:05:47 +08:00
2025-02-05 18:07:53 +08:00
if wc != len(items) {
return fmt.Errorf("got items %d, but wrote %d", len(items), wc)
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
log.Info("Dump: query dump success = %d", wc)
2024-03-22 18:05:47 +08:00
}
2025-02-05 18:07:53 +08:00
return nil
2024-03-22 18:05:47 +08:00
}