diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index e99f8a9..79ce941 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "github.com/loveuer/esgo2dump/log" "github.com/loveuer/esgo2dump/internal/opt" "github.com/spf13/cobra" @@ -48,7 +49,7 @@ esgo2dump --input=http://127.0.0.1:9200/some_index --output=./data.json --query_ ) func init() { - rootCommand.Flags().BoolVar(&opt.Debug, "debug", false, "") + rootCommand.PersistentFlags().BoolVar(&opt.Debug, "debug", false, "") rootCommand.Flags().BoolVarP(&f_version, "version", "v", false, "print esgo2dump version") rootCommand.Flags().IntVar(&opt.Timeout, "timeout", 30, "max timeout seconds per operation with limit") @@ -62,6 +63,12 @@ func init() { rootCommand.Flags().StringVarP(&f_query, "query", "q", "", `query dsl, example: {"bool":{"must":[{"term":{"name":{"value":"some_name"}}}],"must_not":[{"range":{"age":{"gte":18,"lt":60}}}]}}`) rootCommand.Flags().StringVar(&f_query_file, "query_file", "", `query json file (will execute line by line)`) rootCommand.Flags().Uint64VarP(&f_limit, "limit", "l", 100, "") + + rootCommand.PersistentPreRun = func(cmd *cobra.Command, args []string) { + if opt.Debug { + log.SetLogLevel(log.LogLevelDebug) + } + } } func Start(ctx context.Context) error { diff --git a/internal/cmd/run.go b/internal/cmd/run.go index b7ee291..b031449 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -51,10 +51,6 @@ func run(cmd *cobra.Command, args []string) error { ioo interfaces.DumpIO ) - if opt.Debug { - log.SetLogLevel(log.LogLevelDebug) - } - if f_version { fmt.Printf("esgo2dump (Version: %s)\n", opt.Version) os.Exit(0) diff --git a/internal/util/min.go b/internal/util/min.go index 419eb04..b85dd37 100644 --- a/internal/util/min.go +++ b/internal/util/min.go @@ -7,3 +7,15 @@ func Min[T ~string | ~int | ~int64 | ~uint64 | ~float64 | ~float32 | ~int32 | ~u return b } + +func AbsMin(a, b uint64) uint64 { + if a == 0 { + return b + } + + if b == 0 { + return a + } + + return Min(a, b) +} diff --git a/xes/es7/read.go b/xes/es7/read.go index cdb8013..a60967a 100644 --- a/xes/es7/read.go +++ b/xes/es7/read.go @@ -199,11 +199,14 @@ func ReadDataV2( close(errCh) }() + fina_size := util.AbsMin(size, max-total) + log.Debug("es7.read: size = %d, max = %d, total = %d, fina size = %d", size, max, total, fina_size) + for { qs = []func(*esapi.SearchRequest){ client.Search.WithContext(util.TimeoutCtx(ctx, 30)), client.Search.WithIndex(index), - client.Search.WithSize(int(util.Min(size, max-total))), + client.Search.WithSize(int(fina_size)), client.Search.WithSort(sorts...), } @@ -221,6 +224,8 @@ func ReadDataV2( return } + log.Debug("body raw: %s", string(bs)) + qs = append(qs, client.Search.WithBody(bytes.NewReader(bs))) if resp, err = client.Search(qs...); err != nil { errCh <- err @@ -247,6 +252,8 @@ func ReadDataV2( dataCh <- result.Hits.Hits total += uint64(len(result.Hits.Hits)) + log.Debug("es7.read: total: %d", total) + if uint64(len(result.Hits.Hits)) < size || (max > 0 && total >= max) { break }