From 983533237f0956dd2289034f2d52d2eb25f4d5eb Mon Sep 17 00:00:00 2001 From: loveuer Date: Fri, 22 Mar 2024 18:05:47 +0800 Subject: [PATCH] wip: basic feat --- .gitignore | 3 ++ go.mod | 21 ++++++++++ go.sum | 42 +++++++++++++++++++ internal/cmd/init.go | 5 +++ internal/cmd/root.go | 25 +++++++++++ internal/cmd/run.go | 78 +++++++++++++++++++++++++++++++++++ internal/cmd/start.go | 7 ++++ internal/es/client.go | 65 +++++++++++++++++++++++++++++ internal/interfaces/dumpio.go | 7 ++++ internal/opt/var.go | 5 +++ internal/util/ctx.go | 17 ++++++++ internal/xfile/xfile.go | 29 +++++++++++++ main.go | 23 +++++++++++ 13 files changed, 327 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/cmd/init.go create mode 100644 internal/cmd/root.go create mode 100644 internal/cmd/run.go create mode 100644 internal/cmd/start.go create mode 100644 internal/es/client.go create mode 100644 internal/interfaces/dumpio.go create mode 100644 internal/opt/var.go create mode 100644 internal/util/ctx.go create mode 100644 internal/xfile/xfile.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebf869b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +.vscode +.DS_Store \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c5cc898 --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module esgo2dump + +go 1.20 + +require ( + github.com/elastic/go-elasticsearch/v8 v8.12.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.8.0 +) + +require ( + github.com/elastic/elastic-transport-go/v8 v8.4.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect + golang.org/x/sys v0.14.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4335a83 --- /dev/null +++ b/go.sum @@ -0,0 +1,42 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/elastic-transport-go/v8 v8.4.0 h1:EKYiH8CHd33BmMna2Bos1rDNMM89+hdgcymI+KzJCGE= +github.com/elastic/elastic-transport-go/v8 v8.4.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.12.1 h1:QcuFK5LaZS0pSIj/eAEsxmJWmMo7tUs1aVBbzdIgtnE= +github.com/elastic/go-elasticsearch/v8 v8.12.1/go.mod h1:wSzJYrrKPZQ8qPuqAqc6KMR4HrBfHnZORvyL+FMFqq0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/init.go b/internal/cmd/init.go new file mode 100644 index 0000000..fe35f02 --- /dev/null +++ b/internal/cmd/init.go @@ -0,0 +1,5 @@ +package cmd + +func init() { + initRootCommand() +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go new file mode 100644 index 0000000..962a8d5 --- /dev/null +++ b/internal/cmd/root.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "esgo2dump/internal/opt" + "github.com/spf13/cobra" +) + +var ( + rootCommand = &cobra.Command{ + Use: "esgo2dump", + Short: "esgo2dump is alternative to elasticdump", + RunE: run, + } + + input string + output string + limit int +) + +func initRootCommand() { + rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "") + rootCommand.Flags().StringVarP(&input, "input", "i", "https://127.0.0.1:9200", "") + rootCommand.Flags().StringVarP(&output, "output", "o", "output.json", "") + rootCommand.Flags().IntVarP(&limit, "limit", "l", 100, "") +} diff --git a/internal/cmd/run.go b/internal/cmd/run.go new file mode 100644 index 0000000..16ff52c --- /dev/null +++ b/internal/cmd/run.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "esgo2dump/internal/es" + "esgo2dump/internal/interfaces" + "esgo2dump/internal/opt" + "esgo2dump/internal/xfile" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "net/url" + "os" +) + +func run(cmd *cobra.Command, args []string) error { + var ( + err error + ioi interfaces.DumpIO + ioo interfaces.DumpIO + ) + + if opt.Debug { + logrus.SetLevel(logrus.DebugLevel) + } + + if ioi, err = newIO(input); err != nil { + return err + } + + if ioo, err = newIO(output); err != nil { + return err + } + + _ = ioi + _ = ioo + + return nil +} + +func newIO(source string) (interfaces.DumpIO, error) { + var ( + err error + iurl *url.URL + file *os.File + ) + + logrus.Debugf("newIO: source string=%s", source) + + if iurl, err = url.Parse(source); err != nil { + logrus.Debugf("newIO: url parse source err=%v", err) + goto ClientByFile + } + + if iurl.Scheme == "" { + logrus.Debugf("newIO: url scheme default to 'http'") + iurl.Scheme = "http" + } + + if !(iurl.Scheme == "http" || iurl.Scheme == "https") { + logrus.Debugf("newIO: url scheme=%s invalid", iurl.Scheme) + goto ClientByFile + } + + if iurl.Host == "" { + logrus.Debug("newIO: url host empty") + goto ClientByFile + } + + logrus.Debugf("newIO: source as url=%+v", *iurl) + + return es.NewClient(iurl) + +ClientByFile: + if file, err = os.OpenFile(source, os.O_CREATE|os.O_RDWR, 0644); err != nil { + return nil, err + } + + return xfile.NewClient(file) +} diff --git a/internal/cmd/start.go b/internal/cmd/start.go new file mode 100644 index 0000000..c557aa1 --- /dev/null +++ b/internal/cmd/start.go @@ -0,0 +1,7 @@ +package cmd + +import "context" + +func Start(ctx context.Context) error { + return rootCommand.ExecuteContext(ctx) +} diff --git a/internal/es/client.go b/internal/es/client.go new file mode 100644 index 0000000..ab880b2 --- /dev/null +++ b/internal/es/client.go @@ -0,0 +1,65 @@ +package es + +import ( + "esgo2dump/internal/interfaces" + "fmt" + elastic "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/sirupsen/logrus" + "net/url" +) + +func NewClient(url *url.URL) (interfaces.DumpIO, error) { + + var ( + err error + endpoint = fmt.Sprintf("%s://%s:%s", url.Scheme, url.Host, url.Port()) + c *elastic.Client + infoResp *esapi.Response + ) + + logrus.Debugf("es.NewClient: endpoint=%s", endpoint) + + if c, err = elastic.NewClient( + elastic.Config{ + Addresses: []string{endpoint}, + Username: "", + Password: "", + CACert: nil, + RetryOnStatus: []int{429}, + MaxRetries: 3, + RetryBackoff: nil, + }, + ); err != nil { + logrus.Debugf("es.NewClient: elastic new client with endpont=%s err=%v", endpoint, err) + return nil, err + } + + if infoResp, err = c.Info(); err != nil { + return nil, err + } + + if infoResp.StatusCode != 200 { + return nil, fmt.Errorf("info es status=%d", infoResp.StatusCode) + } + + return &client{c: c}, nil +} + +type client struct { + c *elastic.Client +} + +func (c *client) Close() error { + return nil +} + +func (c client) Write(docs []map[string]any) (int, error) { + //TODO implement me + panic("implement me") +} + +func (c client) Read(i int) ([]map[string]any, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/interfaces/dumpio.go b/internal/interfaces/dumpio.go new file mode 100644 index 0000000..5f28c85 --- /dev/null +++ b/internal/interfaces/dumpio.go @@ -0,0 +1,7 @@ +package interfaces + +type DumpIO interface { + Write(docs []map[string]any) (int, error) + Read(int) ([]map[string]any, error) + Close() error +} diff --git a/internal/opt/var.go b/internal/opt/var.go new file mode 100644 index 0000000..a3e7ac4 --- /dev/null +++ b/internal/opt/var.go @@ -0,0 +1,5 @@ +package opt + +var ( + Debug bool +) diff --git a/internal/util/ctx.go b/internal/util/ctx.go new file mode 100644 index 0000000..3e9010d --- /dev/null +++ b/internal/util/ctx.go @@ -0,0 +1,17 @@ +package util + +import ( + "context" + "time" +) + +func Timeout(seconds ...int) context.Context { + second := 30 + if len(seconds) > 0 && seconds[0] > 0 { + second = seconds[0] + } + + ctx, _ := context.WithTimeout(context.Background(), time.Duration(second)*time.Second) + + return ctx +} diff --git a/internal/xfile/xfile.go b/internal/xfile/xfile.go new file mode 100644 index 0000000..4dec3a2 --- /dev/null +++ b/internal/xfile/xfile.go @@ -0,0 +1,29 @@ +package xfile + +import ( + "esgo2dump/internal/interfaces" + "os" +) + +type client struct { + f *os.File +} + +func (c client) Write(docs []map[string]any) (int, error) { + //TODO implement me + panic("implement me") +} + +func (c client) Read(i int) ([]map[string]any, error) { + //TODO implement me + panic("implement me") +} + +func (c client) Close() error { + //TODO implement me + panic("implement me") +} + +func NewClient(file *os.File) (interfaces.DumpIO, error) { + return &client{f: file}, nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e79e17a --- /dev/null +++ b/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "context" + "esgo2dump/internal/cmd" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" +) + +func main() { + logrus.Info("hello world") + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + defer cancel() + + if err := cmd.Start(ctx); err != nil { + logrus.Fatal(err) + } + + logrus.Debug("main: cmd start success!!!") +}