119 lines
2.2 KiB
Go
119 lines
2.2 KiB
Go
package xfile
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/loveuer/esgo2dump/internal/opt"
|
|
"github.com/loveuer/esgo2dump/pkg/log"
|
|
"github.com/loveuer/esgo2dump/pkg/model"
|
|
)
|
|
|
|
type client struct {
|
|
info os.FileInfo
|
|
f *os.File
|
|
scanner *bufio.Scanner
|
|
}
|
|
|
|
// Read implements model.IO.
|
|
func (c *client) ReadData(limit int, query map[string]any, fields []string, sort []string) ([]map[string]any, error) {
|
|
if len(query) != 0 {
|
|
return nil, fmt.Errorf("file with query is unsupported")
|
|
}
|
|
|
|
if len(sort) != 0 {
|
|
return nil, fmt.Errorf("file with sort is unsupported")
|
|
}
|
|
|
|
list := make([]map[string]any, 0, limit)
|
|
|
|
for c.scanner.Scan() {
|
|
line := c.scanner.Bytes()
|
|
item := make(map[string]any)
|
|
|
|
if err := json.Unmarshal(line, &item); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(fields) > 0 {
|
|
// todo: pick fields
|
|
}
|
|
|
|
list = append(list, item)
|
|
|
|
if len(list) >= limit {
|
|
return list, nil
|
|
}
|
|
}
|
|
|
|
return list, nil
|
|
}
|
|
|
|
// Write implements model.IO.
|
|
func (c *client) WriteData(items []map[string]any) (int, error) {
|
|
total := 0
|
|
for _, item := range items {
|
|
bs, err := json.Marshal(item)
|
|
if err != nil {
|
|
return total, err
|
|
}
|
|
|
|
if _, err = c.f.Write(bs); err != nil {
|
|
return total, err
|
|
}
|
|
|
|
total++
|
|
|
|
if _, err = c.f.WriteString("\n"); err != nil {
|
|
return total, err
|
|
}
|
|
}
|
|
|
|
return total, nil
|
|
}
|
|
|
|
func NewClient(path string, t model.IOType) (model.IO[map[string]any], error) {
|
|
var (
|
|
info os.FileInfo
|
|
err error
|
|
f *os.File
|
|
)
|
|
|
|
switch t {
|
|
case model.Input:
|
|
if info, err = os.Stat(path); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
log.Debug("input file: %s, size: %d", path, info.Size())
|
|
|
|
if f, err = os.Open(path); err != nil {
|
|
return nil, err
|
|
}
|
|
case model.Output:
|
|
if info, err = os.Stat(path); err == nil {
|
|
return nil, fmt.Errorf("file already exists: %s", path)
|
|
}
|
|
|
|
if !os.IsNotExist(err) {
|
|
return nil, err
|
|
}
|
|
|
|
if f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
default:
|
|
return nil, fmt.Errorf("unknown type: %s", t)
|
|
}
|
|
|
|
c := &client{f: f, info: info}
|
|
buf := make([]byte, opt.BuffSize)
|
|
scanner := bufio.NewScanner(c.f)
|
|
scanner.Buffer(buf, opt.MaxBuffSize)
|
|
c.scanner = scanner
|
|
|
|
return c, nil
|
|
}
|