refactor: project code arch; feat: cancel operation anytime
This commit is contained in:
loveuer
@ -4,43 +4,74 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/loveuer/esgo2dump/internal/opt"
|
||||
"github.com/loveuer/esgo2dump/model"
|
||||
"github.com/loveuer/nf/nft/log"
|
||||
|
||||
"github.com/loveuer/esgo2dump/internal/interfaces"
|
||||
"github.com/loveuer/esgo2dump/pkg/log"
|
||||
"github.com/loveuer/esgo2dump/pkg/model"
|
||||
)
|
||||
|
||||
type client struct {
|
||||
info os.FileInfo
|
||||
f *os.File
|
||||
iot interfaces.IO
|
||||
scanner *bufio.Scanner
|
||||
}
|
||||
|
||||
func (c *client) WriteData(ctx context.Context, docsCh <-chan []*model.ESSource) error {
|
||||
total := 0
|
||||
for line := range docsCh {
|
||||
for _, doc := range line {
|
||||
bs, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = c.f.Write(append(bs, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
count := len(line)
|
||||
total += count
|
||||
|
||||
log.Info("Dump: succeed=%d total=%d docs succeed!!!", count, total)
|
||||
func (c *client) ReadData(ctx context.Context, limit int, query map[string]any, fields []string, sort []string) ([]map[string]any, error) {
|
||||
if len(query) != 0 {
|
||||
return nil, fmt.Errorf("file with query is unsupported")
|
||||
}
|
||||
|
||||
return nil
|
||||
if len(sort) != 0 {
|
||||
return nil, fmt.Errorf("file with sort is unsupported")
|
||||
}
|
||||
|
||||
list := make([]map[string]any, 0, limit)
|
||||
|
||||
for c.scanner.Scan() {
|
||||
line := c.scanner.Bytes()
|
||||
item := make(map[string]any)
|
||||
|
||||
if err := json.Unmarshal(line, &item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(fields) > 0 {
|
||||
// todo: pick fields
|
||||
}
|
||||
|
||||
list = append(list, item)
|
||||
|
||||
if len(list) >= limit {
|
||||
return list, nil
|
||||
}
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (c *client) WriteData(ctx context.Context, items []map[string]any) (int, error) {
|
||||
total := 0
|
||||
for _, item := range items {
|
||||
bs, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
|
||||
if _, err = c.f.Write(bs); err != nil {
|
||||
return total, err
|
||||
}
|
||||
|
||||
total++
|
||||
|
||||
if _, err = c.f.WriteString("\n"); err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (c *client) ReadMapping(ctx context.Context) (map[string]any, error) {
|
||||
@ -62,6 +93,17 @@ func (c *client) ReadMapping(ctx context.Context) (map[string]any, error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (c *client) WriteMapping(ctx context.Context, mapping map[string]any) error {
|
||||
bs, err := json.Marshal(mapping)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = c.f.Write(bs)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *client) ReadSetting(ctx context.Context) (map[string]any, error) {
|
||||
var (
|
||||
err error
|
||||
@ -81,8 +123,8 @@ func (c *client) ReadSetting(ctx context.Context) (map[string]any, error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (c *client) WriteMapping(ctx context.Context, m map[string]any) error {
|
||||
bs, err := json.Marshal(m)
|
||||
func (c *client) WriteSetting(ctx context.Context, setting map[string]any) error {
|
||||
bs, err := json.Marshal(setting)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -92,99 +134,45 @@ func (c *client) WriteMapping(ctx context.Context, m map[string]any) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *client) WriteSetting(ctx context.Context, m map[string]any) error {
|
||||
bs, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = c.f.Write(bs)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *client) IOType() interfaces.IO {
|
||||
return c.iot
|
||||
}
|
||||
|
||||
func (c *client) IsFile() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *client) ReadData(ctx context.Context, size int, _ map[string]any, _ []string, _ []string) (<-chan []*model.ESSource, <-chan error) {
|
||||
func NewClient(path string, t model.IOType) (model.IO[map[string]any], error) {
|
||||
var (
|
||||
err error
|
||||
count int = 0
|
||||
list = make([]*model.ESSource, 0, size)
|
||||
dch = make(chan []*model.ESSource)
|
||||
ech = make(chan error)
|
||||
ready = make(chan bool)
|
||||
total = 0
|
||||
info os.FileInfo
|
||||
err error
|
||||
f *os.File
|
||||
)
|
||||
|
||||
go func(ctx context.Context) {
|
||||
defer func() {
|
||||
close(dch)
|
||||
close(ech)
|
||||
}()
|
||||
|
||||
ready <- true
|
||||
|
||||
for c.scanner.Scan() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
item := new(model.ESSource)
|
||||
line := c.scanner.Bytes()
|
||||
|
||||
if err = json.Unmarshal(line, item); err != nil {
|
||||
ech <- err
|
||||
return
|
||||
}
|
||||
|
||||
list = append(list, item)
|
||||
count++
|
||||
total++
|
||||
|
||||
if count >= size {
|
||||
dch <- list
|
||||
list = list[:0]
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
switch t {
|
||||
case model.Input:
|
||||
if info, err = os.Stat(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(list) > 0 {
|
||||
dch <- list
|
||||
list = list[:0]
|
||||
count = 0
|
||||
log.Debug("input file: %s, size: %d", path, info.Size())
|
||||
|
||||
if f, err = os.Open(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case model.Output:
|
||||
if info, err = os.Stat(path); err == nil {
|
||||
return nil, fmt.Errorf("file already exists: %s", path)
|
||||
}
|
||||
|
||||
if err = c.scanner.Err(); err != nil {
|
||||
ech <- err
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("read: read file succeed! total=%d", total)
|
||||
}(ctx)
|
||||
|
||||
<-ready
|
||||
|
||||
return dch, ech
|
||||
}
|
||||
|
||||
func (c *client) Close() error {
|
||||
return c.f.Close()
|
||||
}
|
||||
|
||||
func NewClient(file *os.File, ioType interfaces.IO) (interfaces.DumpIO, error) {
|
||||
c := &client{f: file, iot: ioType}
|
||||
|
||||
if ioType == interfaces.IOInput {
|
||||
c.scanner = bufio.NewScanner(c.f)
|
||||
buf := make([]byte, opt.BuffSize)
|
||||
c.scanner.Buffer(buf, opt.MaxBuffSize)
|
||||
if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown type: %s", t)
|
||||
}
|
||||
|
||||
c := &client{f: f, info: info}
|
||||
buf := make([]byte, opt.BuffSize)
|
||||
scanner := bufio.NewScanner(c.f)
|
||||
scanner.Buffer(buf, opt.MaxBuffSize)
|
||||
c.scanner = scanner
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user