wip: basic feat

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.vscode
+.DS_Store

go.mod

@@ -0,0 +1,21 @@
+module esgo2dump
+
+go 1.20
+
+require (
+	github.com/elastic/go-elasticsearch/v8 v8.12.1
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
+)
+
+require (
+	github.com/elastic/elastic-transport-go/v8 v8.4.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+)

go.sum

@@ -0,0 +1,42 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/elastic-transport-go/v8 v8.4.0 h1:EKYiH8CHd33BmMna2Bos1rDNMM89+hdgcymI+KzJCGE=
+github.com/elastic/elastic-transport-go/v8 v8.4.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
+github.com/elastic/go-elasticsearch/v8 v8.12.1 h1:QcuFK5LaZS0pSIj/eAEsxmJWmMo7tUs1aVBbzdIgtnE=
+github.com/elastic/go-elasticsearch/v8 v8.12.1/go.mod h1:wSzJYrrKPZQ8qPuqAqc6KMR4HrBfHnZORvyL+FMFqq0=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
+go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
+go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/cmd/init.go

@@ -0,0 +1,5 @@
+package cmd
+
+func init() {
+	initRootCommand()
+}

internal/cmd/root.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"esgo2dump/internal/opt"
+	"github.com/spf13/cobra"
+)
+
+var (
+	rootCommand = &cobra.Command{
+		Use:   "esgo2dump",
+		Short: "esgo2dump is alternative to elasticdump",
+		RunE:  run,
+	}
+
+	input  string
+	output string
+	limit  int
+)
+
+func initRootCommand() {
+	rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "")
+	rootCommand.Flags().StringVarP(&input, "input", "i", "https://127.0.0.1:9200", "")
+	rootCommand.Flags().StringVarP(&output, "output", "o", "output.json", "")
+	rootCommand.Flags().IntVarP(&limit, "limit", "l", 100, "")
+}

internal/cmd/run.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"esgo2dump/internal/es"
+	"esgo2dump/internal/interfaces"
+	"esgo2dump/internal/opt"
+	"esgo2dump/internal/xfile"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"net/url"
+	"os"
+)
+
+func run(cmd *cobra.Command, args []string) error {
+	var (
+		err error
+		ioi interfaces.DumpIO
+		ioo interfaces.DumpIO
+	)
+
+	if opt.Debug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
+	if ioi, err = newIO(input); err != nil {
+		return err
+	}
+
+	if ioo, err = newIO(output); err != nil {
+		return err
+	}
+
+	_ = ioi
+	_ = ioo
+
+	return nil
+}
+
+func newIO(source string) (interfaces.DumpIO, error) {
+	var (
+		err  error
+		iurl *url.URL
+		file *os.File
+	)
+
+	logrus.Debugf("newIO: source string=%s", source)
+
+	if iurl, err = url.Parse(source); err != nil {
+		logrus.Debugf("newIO: url parse source err=%v", err)
+		goto ClientByFile
+	}
+
+	if iurl.Scheme == "" {
+		logrus.Debugf("newIO: url scheme default to 'http'")
+		iurl.Scheme = "http"
+	}
+
+	if !(iurl.Scheme == "http" || iurl.Scheme == "https") {
+		logrus.Debugf("newIO: url scheme=%s invalid", iurl.Scheme)
+		goto ClientByFile
+	}
+
+	if iurl.Host == "" {
+		logrus.Debug("newIO: url host empty")
+		goto ClientByFile
+	}
+
+	logrus.Debugf("newIO: source as url=%+v", *iurl)
+
+	return es.NewClient(iurl)
+
+ClientByFile:
+	if file, err = os.OpenFile(source, os.O_CREATE|os.O_RDWR, 0644); err != nil {
+		return nil, err
+	}
+
+	return xfile.NewClient(file)
+}

internal/cmd/start.go

@@ -0,0 +1,7 @@
+package cmd
+
+import "context"
+
+func Start(ctx context.Context) error {
+	return rootCommand.ExecuteContext(ctx)
+}

internal/es/client.go

@@ -0,0 +1,65 @@
+package es
+
+import (
+	"esgo2dump/internal/interfaces"
+	"fmt"
+	elastic "github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/esapi"
+	"github.com/sirupsen/logrus"
+	"net/url"
+)
+
+func NewClient(url *url.URL) (interfaces.DumpIO, error) {
+
+	var (
+		err      error
+		endpoint = fmt.Sprintf("%s://%s:%s", url.Scheme, url.Host, url.Port())
+		c        *elastic.Client
+		infoResp *esapi.Response
+	)
+
+	logrus.Debugf("es.NewClient: endpoint=%s", endpoint)
+
+	if c, err = elastic.NewClient(
+		elastic.Config{
+			Addresses:     []string{endpoint},
+			Username:      "",
+			Password:      "",
+			CACert:        nil,
+			RetryOnStatus: []int{429},
+			MaxRetries:    3,
+			RetryBackoff:  nil,
+		},
+	); err != nil {
+		logrus.Debugf("es.NewClient: elastic new client with endpont=%s err=%v", endpoint, err)
+		return nil, err
+	}
+
+	if infoResp, err = c.Info(); err != nil {
+		return nil, err
+	}
+
+	if infoResp.StatusCode != 200 {
+		return nil, fmt.Errorf("info es status=%d", infoResp.StatusCode)
+	}
+
+	return &client{c: c}, nil
+}
+
+type client struct {
+	c *elastic.Client
+}
+
+func (c *client) Close() error {
+	return nil
+}
+
+func (c client) Write(docs []map[string]any) (int, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (c client) Read(i int) ([]map[string]any, error) {
+	//TODO implement me
+	panic("implement me")
+}

internal/interfaces/dumpio.go

@@ -0,0 +1,7 @@
+package interfaces
+
+type DumpIO interface {
+	Write(docs []map[string]any) (int, error)
+	Read(int) ([]map[string]any, error)
+	Close() error
+}

internal/opt/var.go

@@ -0,0 +1,5 @@
+package opt
+
+var (
+	Debug bool
+)

internal/util/ctx.go

@@ -0,0 +1,17 @@
+package util
+
+import (
+	"context"
+	"time"
+)
+
+func Timeout(seconds ...int) context.Context {
+	second := 30
+	if len(seconds) > 0 && seconds[0] > 0 {
+		second = seconds[0]
+	}
+
+	ctx, _ := context.WithTimeout(context.Background(), time.Duration(second)*time.Second)
+
+	return ctx
+}

internal/xfile/xfile.go

@@ -0,0 +1,29 @@
+package xfile
+
+import (
+	"esgo2dump/internal/interfaces"
+	"os"
+)
+
+type client struct {
+	f *os.File
+}
+
+func (c client) Write(docs []map[string]any) (int, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (c client) Read(i int) ([]map[string]any, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (c client) Close() error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func NewClient(file *os.File) (interfaces.DumpIO, error) {
+	return &client{f: file}, nil
+}

main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"context"
+	"esgo2dump/internal/cmd"
+	"os/signal"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	logrus.Info("hello world")
+
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	defer cancel()
+
+	if err := cmd.Start(ctx); err != nil {
+		logrus.Fatal(err)
+	}
+
+	logrus.Debug("main: cmd start success!!!")
+}