loveuer/esgo2dump
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: dump from es with query, limit

Browse Source
This commit is contained in:
loveuer
2024-03-23 21:25:19 +08:00
parent 983533237f
commit f1af294ced
16 changed files with 446 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/cmd/cmd.go Normal file
View File

@ -0,0 +1,44 @@
package cmd
import (
"context"
"esgo2dump/internal/opt"
"github.com/spf13/cobra"
)
var (
rootCommand = &cobra.Command{
Use: "esgo2dump",
Short: "esgo2dump is alternative to elasticdump",
SilenceUsage: true,
SilenceErrors: true,
RunE: run,
Example: `
esgo2dump --input=http://127.0.0.1:9200/some_index --output=./data.json
esgo2dump --input=https://username:password@127.0.0.1:9200/some_index --output=./data.json
esgo2dump --input=http://127.0.0.1:9200/some_index --output=./data.json --query='{"match": {"name": "some_name"}}'`,
}
f_input string
f_output string
f_limit int
f_type string
f_query string
)
func init() {
rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "")
rootCommand.Flags().IntVar(&opt.Timeout, "timeout", 30, "max timeout seconds per operation with limit")
rootCommand.Flags().StringVarP(&f_input, "input", "i", "http://127.0.0.1:9200/my_index", "")
rootCommand.Flags().StringVarP(&f_output, "output", "o", "output.json", "")
rootCommand.Flags().StringVarP(&f_type, "type", "t", "data", "data/mapping/setting")
rootCommand.Flags().StringVarP(&f_query, "query", "q", "", `query dsl, example: {"bool":{"must":[{"term":{"name":{"value":"some_name"}}}],"must_not":[{"range":{"age":{"gte":18,"lt":60}}}]}}`)
rootCommand.Flags().IntVarP(&f_limit, "limit", "l", 100, "")
}
func Start(ctx context.Context) error {
return rootCommand.ExecuteContext(ctx)
}

5
internal/cmd/init.go
View File

@ -1,5 +0,0 @@
package cmd
func init() {
initRootCommand()
}

25
internal/cmd/root.go
View File

@ -1,25 +0,0 @@
package cmd
import (
"esgo2dump/internal/opt"
"github.com/spf13/cobra"
)
var (
rootCommand = &cobra.Command{
Use: "esgo2dump",
Short: "esgo2dump is alternative to elasticdump",
RunE: run,
}
input string
output string
limit int
)
func initRootCommand() {
rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "")
rootCommand.Flags().StringVarP(&input, "input", "i", "https://127.0.0.1:9200", "")
rootCommand.Flags().StringVarP(&output, "output", "o", "output.json", "")
rootCommand.Flags().IntVarP(&limit, "limit", "l", 100, "")
}

85
internal/cmd/run.go
View File

@ -1,78 +1,119 @@
package cmd
import (
"esgo2dump/internal/es"
"encoding/json"
"errors"
"esgo2dump/internal/interfaces"
"esgo2dump/internal/opt"
"esgo2dump/internal/xes"
"esgo2dump/internal/xfile"
"fmt"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"io"
"net/url"
"os"
)
func run(cmd *cobra.Command, args []string) error {
var (
err error
ioi interfaces.DumpIO
ioo interfaces.DumpIO
err error
ioi interfaces.DumpIO
ioo interfaces.DumpIO
lines []*interfaces.ESSource
succeed int
)
if opt.Debug {
logrus.SetLevel(logrus.DebugLevel)
}
if ioi, err = newIO(input); err != nil {
if ioi, err = newIO(f_input, interfaces.IOInput); err != nil {
return err
}
if ioo, err = newIO(output); err != nil {
if ioo, err = newIO(f_output, interfaces.IOOutput); err != nil {
return err
}
_ = ioi
_ = ioo
for {
select {
case <-cmd.Context().Done():
default:
succeed = 0
return nil
if lines, err = ioi.Read(f_limit); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
if len(lines) == 0 {
return nil
}
if succeed, err = ioo.Write(lines); err != nil {
return err
}
if succeed != len(lines) {
return fmt.Errorf("cmd.run: got lines=%d, only succeed=%d", len(lines), succeed)
}
logrus.Infof("Dump: %d docs succeed!!!", succeed)
}
}
}
func newIO(source string) (interfaces.DumpIO, error) {
func newIO(source string, ioType interfaces.IO) (interfaces.DumpIO, error) {
var (
err error
iurl *url.URL
file *os.File
qm = make(map[string]any)
)
logrus.Debugf("newIO: source string=%s", source)
logrus.Debugf("newIO.%s: source string=%s", ioType.Code(), source)
if iurl, err = url.Parse(source); err != nil {
logrus.Debugf("newIO: url parse source err=%v", err)
logrus.Debugf("newIO.%s: url parse source err=%v", ioType.Code(), err)
goto ClientByFile
}
if iurl.Scheme == "" {
logrus.Debugf("newIO: url scheme default to 'http'")
iurl.Scheme = "http"
}
if !(iurl.Scheme == "http" || iurl.Scheme == "https") {
logrus.Debugf("newIO: url scheme=%s invalid", iurl.Scheme)
logrus.Debugf("newIO.%s: url scheme=%s invalid", ioType.Code(), iurl.Scheme)
goto ClientByFile
}
if iurl.Host == "" {
logrus.Debug("newIO: url host empty")
logrus.Debugf("newIO.%s: url host empty", ioType.Code())
goto ClientByFile
}
logrus.Debugf("newIO: source as url=%+v", *iurl)
if ioType == interfaces.IOInput && f_query != "" {
if err = json.Unmarshal([]byte(f_query), &qm); err != nil {
logrus.Debugf("newIO.%s: query=%s invalid to map[string]any", ioType.Code(), f_query)
return nil, fmt.Errorf("invalid query err=%v", err)
}
}
return es.NewClient(iurl)
logrus.Debugf("newIO.%s: source as url=%+v", ioType.Code(), *iurl)
return xes.NewClient(iurl, qm)
ClientByFile:
if ioType == interfaces.IOOutput {
if _, err = os.Stat(source); !os.IsNotExist(err) {
return nil, fmt.Errorf("output_file=%s already exist", source)
}
}
if file, err = os.OpenFile(source, os.O_CREATE|os.O_RDWR, 0644); err != nil {
return nil, err
}
return xfile.NewClient(file)
return xfile.NewClient(file, ioType)
}

7
internal/cmd/start.go
View File

@ -1,7 +0,0 @@
package cmd
import "context"
func Start(ctx context.Context) error {
return rootCommand.ExecuteContext(ctx)
}