loveuer/nfflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: flow task start

Browse Source
This commit is contained in:
loveuer
2024-04-03 16:46:25 +08:00
parent 4718532458
commit bb56271348
11 changed files with 381 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/controller/impl.go
View File

@ -1,6 +1,9 @@
package controller
import "github.com/loveuer/nfflow/internal/database"
import (
"github.com/loveuer/nfflow/internal/database"
"github.com/loveuer/nfflow/internal/interfaces"
)
type uc struct {
db database.Store
@ -8,10 +11,10 @@ type uc struct {
}
var (
_ userController = uc{}
_ interfaces.UserController = (*uc)(nil)
// UserController todo: 可以实现自己的 controller
UserController userController
UserController interfaces.UserController
)
func Init(db database.Store, cache database.Caches) {

74
internal/controller/input/es7/es7.go
View File

@ -7,8 +7,10 @@ import (
"fmt"
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/loveuer/nfflow/internal/interfaces"
"github.com/loveuer/nfflow/internal/model"
"github.com/loveuer/nfflow/internal/opt"
"github.com/loveuer/nfflow/internal/sqlType"
"github.com/loveuer/nfflow/internal/util"
"github.com/sirupsen/logrus"
)
@ -16,23 +18,52 @@ import (
type ES7 struct {
cli *elastic.Client
scroll string
cfg struct {
Endpoints []string
Username string
Password string
Size int
Query map[string]any
Source []string
}
query map[string]any
size int
max int
source []string
}
func (e *ES7) init() error {
func NewInput(cfg sqlType.JSONB) (interfaces.Input, error) {
type config struct {
Endpoints []string `json:"endpoints"`
Username string `json:"username"`
Password string `json:"password"`
Size int `json:"size"`
Max int `json:"max"`
Query map[string]any `json:"query"`
Source []string `json:"source"`
}
var (
err error
c = new(config)
ins = &ES7{}
)
if err = cfg.Bind(c); err != nil {
return nil, err
}
if err = ins.Init(c.Endpoints, c.Username, c.Password); err != nil {
return nil, err
}
ins.query = c.Query
ins.size = c.Size
ins.max = c.Max
ins.source = c.Source
return ins, nil
}
func (e *ES7) Init(endpoints []string, username, password string) error {
var (
err error
cfg = elastic.Config{
Addresses: e.cfg.Endpoints,
Username: e.cfg.Username,
Password: e.cfg.Password,
Addresses: endpoints,
Username: username,
Password: password,
RetryOnStatus: []int{429},
}
info *esapi.Response
@ -53,7 +84,7 @@ func (e *ES7) init() error {
return nil
}
func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.TaskRow, errCh chan<- error) error {
func (e *ES7) Start(ctx context.Context, rowCh chan<- interfaces.Row, errCh chan<- error) error {
var (
err error
result *esapi.Response
@ -63,20 +94,15 @@ func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.Ta
hits = new(model.ESResponse)
)
if err = e.init(); err != nil {
logrus.Debugf("ES7.Start: init err=%v", err)
return err
}
qs := []func(*esapi.SearchRequest){
e.cli.Search.WithContext(util.TimeoutCtx(ctx, opt.ES7OperationTimeout)),
e.cli.Search.WithScroll(opt.ScrollTimeout),
e.cli.Search.WithSize(e.cfg.Size),
e.cli.Search.WithSize(e.size),
}
if e.cfg.Query != nil && len(e.cfg.Query) > 0 {
if e.query != nil && len(e.query) > 0 {
var bs []byte
if bs, err = json.Marshal(e.cfg.Query); err != nil {
if bs, err = json.Marshal(map[string]any{"query": e.query}); err != nil {
logrus.Debugf("ES7.Start: marshal query err=%v", err)
return err
}
@ -139,7 +165,7 @@ func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.Ta
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
if len(hits.Hits.Hits) < e.size {
return
}
@ -179,7 +205,7 @@ func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.Ta
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
if len(hits.Hits.Hits) < e.size {
return
}
}
@ -189,3 +215,5 @@ func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.Ta
return nil
}
func (e *ES7) Close() {}

12
internal/controller/interface.go
View File

@ -1,12 +0,0 @@
package controller
import "github.com/loveuer/nfflow/internal/model"
type userController interface {
GetUser(id uint64) (*model.User, error)
GetUserByToken(token string) (*model.User, error)
CacheUser(user *model.User) error
CacheToken(token string, user *model.User) error
RmUserCache(id uint64) error
DeleteUser(id uint64) error
}

80
internal/controller/output/xfile/local.go
View File

@ -2,44 +2,78 @@ package xfile
import (
"context"
"fmt"
"github.com/loveuer/nfflow/internal/model"
"io"
"errors"
"github.com/loveuer/nfflow/internal/interfaces"
"github.com/loveuer/nfflow/internal/sqlType"
"github.com/sirupsen/logrus"
"os"
)
type LocalFile struct {
writer io.Writer
cfg struct {
MaxSize int
Path string
}
writer *os.File
Path string `json:"path"`
MaxLine int `json:"max_line"`
}
func (lf *LocalFile) init() error {
func NewFileOutput(cfg sqlType.JSONB) (interfaces.Output, error) {
var (
err error
ins = &LocalFile{}
)
if _, err = os.Stat(lf.cfg.Path); !os.IsNotExist(err) {
return fmt.Errorf("file=%s already exist", lf.cfg.Path)
if err = cfg.Bind(ins); err != nil {
return nil, err
}
if lf.writer, err = os.OpenFile(lf.cfg.Path, os.O_CREATE|os.O_RDWR, 0644); err != nil {
return fmt.Errorf("openfile=%s err=%v", lf.cfg.Path, err)
if _, err = os.Stat(ins.Path); !errors.Is(err, os.ErrNotExist) {
return nil, errors.New("file already exist")
}
if ins.writer, err = os.OpenFile(ins.Path, os.O_CREATE|os.O_RDWR, 0644); err != nil {
return nil, err
}
return ins, nil
}
func (lf *LocalFile) Start(ctx context.Context, rowCh <-chan interfaces.Row, errCh chan<- error) error {
var (
err error
bs []byte
ready = make(chan bool)
)
go func() {
ready <- true
for {
select {
case <-ctx.Done():
logrus.Warn("received quit signal...")
return
case row, ok := <-rowCh:
if !ok {
return
}
if bs, err = row.Bytes(); err != nil {
errCh <- err
return
}
if _, err = lf.writer.Write(append(bs, '\n')); err != nil {
errCh <- err
return
}
}
}
}()
<-ready
return nil
}
func (lf *LocalFile) Start(ctx context.Context, rowCh <-chan *model.TaskRow, errCh chan<- error) error {
var (
err error
)
if err = lf.init(); err != nil {
return err
}
return nil
func (lf *LocalFile) Close() {
_ = lf.writer.Close()
}

22
internal/controller/pipe/loadash/loadash.go
View File

@ -1,29 +1,41 @@
package loadash
import (
"errors"
"fmt"
"github.com/dop251/goja"
"github.com/loveuer/nfflow/internal/interfaces"
"github.com/loveuer/nfflow/internal/sqlType"
"github.com/sirupsen/logrus"
)
type PipeMap struct {
vm *goja.Runtime
FN string `json:"fn"`
}
func NewPipeMap(fn string) (*PipeMap, error) {
func NewPipeMap(cfg sqlType.JSONB) (interfaces.Pipe, error) {
var (
err error
pm = &PipeMap{}
prefix = "let result; let doc; let myFn;"
)
if err = cfg.Bind(pm); err != nil {
return nil, err
}
if pm.FN == "" {
return nil, errors.New("nil cfg")
}
pm.vm = goja.New()
if _, err = pm.vm.RunString(prefix); err != nil {
logrus.Debugf("NewPipeMap: run prepare=%s err=%v", prefix, err)
return nil, err
}
fn = "myFn = " + fn + ";"
fn := "myFn = " + pm.FN + ";"
if _, err = pm.vm.RunString(fn); err != nil {
err = fmt.Errorf("PipeMap: vm run string=%s err=%v", fn, err)
logrus.Error(err)
@ -33,7 +45,7 @@ func NewPipeMap(fn string) (*PipeMap, error) {
return pm, nil
}
func (m *PipeMap) Pipe(data any) (any, error) {
func (m *PipeMap) Start(data any) (any, error) {
var (
err error
)
@ -50,3 +62,7 @@ func (m *PipeMap) Pipe(data any) (any, error) {
return value, nil
}
func (m *PipeMap) Close() {
m.vm = nil
}

188
internal/controller/task.go
View File

@ -1,7 +1,191 @@
package controller
import "github.com/loveuer/nfflow/internal/model"
import (
"context"
"errors"
"github.com/loveuer/nfflow/internal/controller/input/es7"
"github.com/loveuer/nfflow/internal/controller/output/xfile"
"github.com/loveuer/nfflow/internal/controller/pipe/loadash"
"github.com/loveuer/nfflow/internal/database"
"github.com/loveuer/nfflow/internal/interfaces"
"github.com/loveuer/nfflow/internal/model"
"github.com/loveuer/nfflow/internal/util"
"github.com/samber/lo"
"gorm.io/gorm"
"sort"
)
func TaskInitInput(tx *gorm.DB, t *model.Task) (interfaces.Input, error) {
var (
err error
ti = new(model.TaskInput)
)
if err = tx.Model(&model.TaskInput{}).
Where("task_id", t.Id).
Take(ti).
Error; err != nil {
return nil, err
}
switch ti.Type {
case model.InputTypeES:
if ti.Instance, err = es7.NewInput(ti.Config); err != nil {
return nil, err
}
case model.InputTypePG:
panic("not impl input:pg")
case model.InputTypeMQ:
panic("not impl input:mq")
default:
panic("invalid input type")
}
return ti.Instance, nil
}
func TaskInitPipes(tx *gorm.DB, t *model.Task) ([]interfaces.Pipe, error) {
var (
err error
list = make([]*model.TaskPipe, 0)
)
if err = tx.Model(&model.TaskPipe{}).
Where("task_id", t.Id).
Find(&list).
Error; err != nil {
return nil, err
}
for idx := range list {
switch list[idx].Type {
case model.PipeTypeLoadashMap:
if list[idx].Instance, err = loadash.NewPipeMap(list[idx].Config); err != nil {
return nil, err
}
default:
panic("invalid pipe type")
}
}
sort.Slice(list, func(i, j int) bool {
return list[i].Sort < list[j].Sort
})
return lo.Map(list, func(item *model.TaskPipe, index int) interfaces.Pipe {
return item.Instance
}), nil
}
func TaskInitOutputs(tx *gorm.DB, t *model.Task) ([]interfaces.Output, error) {
var (
err error
outputs = make([]*model.TaskOutput, 0)
)
if err = tx.Model(&model.TaskOutput{}).
Where("task_id", t.Id).
Find(&outputs).
Error; err != nil {
return nil, err
}
for _, o := range outputs {
switch o.Type {
case model.OutputTypeES:
panic("impl output es")
case model.OutputTypeFile:
if o.Instance, err = xfile.NewFileOutput(o.Config); err != nil {
return nil, err
}
case model.OutputTypeMQ:
panic("impl output mq")
}
}
return lo.Map(outputs, func(item *model.TaskOutput, index int) interfaces.Output {
return item.Instance
}), nil
}
func TaskStart(ctx context.Context, t *model.Task) error {
var (
err error
input interfaces.Input
pipes []interfaces.Pipe
outputs []interfaces.Output
irc = make(chan interfaces.Row)
prc = make(chan interfaces.Row)
errCh = make(chan error)
done = make(chan bool)
)
if input, err = TaskInitInput(database.DB.Session(util.Timeout(5)), t); err != nil {
return err
}
if input == nil {
return errors.New("nil input")
}
if pipes, err = TaskInitPipes(database.DB.Session(util.Timeout(10)), t); err != nil {
return err
}
if outputs, err = TaskInitOutputs(database.DB.Session(util.Timeout(10)), t); err != nil {
return err
}
if len(outputs) == 0 {
return errors.New("nil output")
}
go func() {
for data := range irc {
var hd interfaces.Row
for idx := range pipes {
if hd, err = pipes[idx].Start(data); err != nil {
errCh <- err
return
}
}
prc <- hd
}
}()
if err = input.Start(ctx, irc, errCh); err != nil {
return err
}
go func() {
chs := make([]chan interfaces.Row, len(outputs))
for idx := range outputs {
ch := make(chan interfaces.Row)
if err = outputs[idx].Start(ctx, ch, errCh); err != nil {
errCh <- err
return
}
}
for od := range prc {
for idx := range chs {
chs[idx] <- od
}
}
done <- true
}()
select {
case err = <-errCh:
return err
case <-done:
}
func TaskStart(t *model.Task) error {
return nil
}