feat: 🎉 完成基本功能
This commit is contained in:
loveuer
46
internal/cmd/cmd.go
Normal file
46
internal/cmd/cmd.go
Normal file
@ -0,0 +1,46 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"esgo2dump/internal/opt"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
rootCommand = &cobra.Command{
|
||||
Use: "esgo2dump",
|
||||
Short: "esgo2dump is alternative to elasticdump",
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
RunE: run,
|
||||
Example: `
|
||||
esgo2dump --input=http://127.0.0.1:9200/some_index --output=./data.json
|
||||
|
||||
esgo2dump --input=http://127.0.0.1:9200/some_index --output=http://192.168.1.1:9200/some_index --limit=5000
|
||||
|
||||
esgo2dump --input=https://username:password@127.0.0.1:9200/some_index --output=./data.json
|
||||
|
||||
esgo2dump --input=http://127.0.0.1:9200/some_index --output=./data.json --query='{"match": {"name": "some_name"}}'`,
|
||||
}
|
||||
|
||||
f_input string
|
||||
f_output string
|
||||
f_limit int
|
||||
f_type string
|
||||
f_query string
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "")
|
||||
rootCommand.Flags().IntVar(&opt.Timeout, "timeout", 30, "max timeout seconds per operation with limit")
|
||||
|
||||
rootCommand.Flags().StringVarP(&f_input, "input", "i", "http://127.0.0.1:9200/my_index", "")
|
||||
rootCommand.Flags().StringVarP(&f_output, "output", "o", "output.json", "")
|
||||
rootCommand.Flags().StringVarP(&f_type, "type", "t", "data", "data/mapping/setting")
|
||||
rootCommand.Flags().StringVarP(&f_query, "query", "q", "", `query dsl, example: {"bool":{"must":[{"term":{"name":{"value":"some_name"}}}],"must_not":[{"range":{"age":{"gte":18,"lt":60}}}]}}`)
|
||||
rootCommand.Flags().IntVarP(&f_limit, "limit", "l", 100, "")
|
||||
}
|
||||
|
||||
func Start(ctx context.Context) error {
|
||||
return rootCommand.ExecuteContext(ctx)
|
||||
}
|
||||
153
internal/cmd/run.go
Normal file
153
internal/cmd/run.go
Normal file
@ -0,0 +1,153 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"esgo2dump/internal/interfaces"
|
||||
"esgo2dump/internal/opt"
|
||||
"esgo2dump/internal/xes"
|
||||
"esgo2dump/internal/xfile"
|
||||
"fmt"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
)
|
||||
|
||||
func run(cmd *cobra.Command, args []string) error {
|
||||
var (
|
||||
err error
|
||||
ioi interfaces.DumpIO
|
||||
ioo interfaces.DumpIO
|
||||
)
|
||||
|
||||
if opt.Debug {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
}
|
||||
|
||||
switch f_type {
|
||||
case "data", "mapping", "setting":
|
||||
default:
|
||||
return fmt.Errorf("unknown type=%s", f_type)
|
||||
}
|
||||
|
||||
if ioi, err = newIO(f_input, interfaces.IOInput); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ioo, err = newIO(f_output, interfaces.IOOutput); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = ioi.Close()
|
||||
_ = ioo.Close()
|
||||
}()
|
||||
|
||||
switch f_type {
|
||||
case "data":
|
||||
return executeData(cmd.Context(), ioi, ioo)
|
||||
case "mapping":
|
||||
var mapping map[string]any
|
||||
if mapping, err = ioi.ReadMapping(cmd.Context()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ioo.WriteMapping(cmd.Context(), mapping)
|
||||
case "setting":
|
||||
var setting map[string]any
|
||||
if setting, err = ioi.ReadSetting(cmd.Context()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ioo.WriteSetting(cmd.Context(), setting)
|
||||
default:
|
||||
return fmt.Errorf("unknown type=%s", f_type)
|
||||
}
|
||||
}
|
||||
|
||||
func executeData(ctx context.Context, input, output interfaces.DumpIO) error {
|
||||
var (
|
||||
err error
|
||||
lines []*interfaces.ESSource
|
||||
succeed int
|
||||
)
|
||||
|
||||
for {
|
||||
|
||||
if lines, err = input.ReadData(ctx, f_limit); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if len(lines) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if succeed, err = output.WriteData(ctx, lines); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if succeed != len(lines) {
|
||||
return fmt.Errorf("cmd.run: got lines=%d, only succeed=%d", len(lines), succeed)
|
||||
}
|
||||
|
||||
logrus.Infof("Dump: %d docs succeed!!!", succeed)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func newIO(source string, ioType interfaces.IO) (interfaces.DumpIO, error) {
|
||||
var (
|
||||
err error
|
||||
iurl *url.URL
|
||||
file *os.File
|
||||
qm = make(map[string]any)
|
||||
)
|
||||
|
||||
logrus.Debugf("newIO.%s: source string=%s", ioType.Code(), source)
|
||||
|
||||
if iurl, err = url.Parse(source); err != nil {
|
||||
logrus.Debugf("newIO.%s: url parse source err=%v", ioType.Code(), err)
|
||||
goto ClientByFile
|
||||
}
|
||||
|
||||
if !(iurl.Scheme == "http" || iurl.Scheme == "https") {
|
||||
logrus.Debugf("newIO.%s: url scheme=%s invalid", ioType.Code(), iurl.Scheme)
|
||||
goto ClientByFile
|
||||
}
|
||||
|
||||
if iurl.Host == "" {
|
||||
logrus.Debugf("newIO.%s: url host empty", ioType.Code())
|
||||
goto ClientByFile
|
||||
}
|
||||
|
||||
if ioType == interfaces.IOInput && f_query != "" {
|
||||
if err = json.Unmarshal([]byte(f_query), &qm); err != nil {
|
||||
logrus.Debugf("newIO.%s: query=%s invalid to map[string]any", ioType.Code(), f_query)
|
||||
return nil, fmt.Errorf("invalid query err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
logrus.Debugf("newIO.%s: source as url=%+v", ioType.Code(), *iurl)
|
||||
|
||||
return xes.NewClient(iurl, ioType, qm)
|
||||
|
||||
ClientByFile:
|
||||
if ioType == interfaces.IOOutput {
|
||||
if _, err = os.Stat(source); !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("output_file=%s already exist", source)
|
||||
}
|
||||
}
|
||||
|
||||
if file, err = os.OpenFile(source, os.O_CREATE|os.O_RDWR, 0644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return xfile.NewClient(file, ioType)
|
||||
}
|
||||
Reference in New Issue
Block a user