loveuer/nfflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: output, pipe

Browse Source
This commit is contained in:
loveuer
2024-04-01 18:13:10 +08:00
parent 195fbcd308
commit 4c40041b3d
16 changed files with 445 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
internal/controller/input/es7/es7.go Normal file
View File

@ -0,0 +1,191 @@
package es7
import (
"bytes"
"context"
"encoding/json"
"fmt"
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/loveuer/nfflow/internal/model"
"github.com/loveuer/nfflow/internal/opt"
"github.com/loveuer/nfflow/internal/util"
"github.com/sirupsen/logrus"
)
type ES7 struct {
cli *elastic.Client
scroll string
cfg struct {
Endpoints []string
Username string
Password string
Size int
Query map[string]any
Source []string
}
}
func (e *ES7) init() error {
var (
err error
cfg = elastic.Config{
Addresses: e.cfg.Endpoints,
Username: e.cfg.Username,
Password: e.cfg.Password,
RetryOnStatus: []int{429},
}
info *esapi.Response
)
if e.cli, err = elastic.NewClient(cfg); err != nil {
return err
}
if info, err = e.cli.Info(e.cli.Info.WithContext(util.Timeout(5))); err != nil {
return err
}
if info.StatusCode != 200 {
return fmt.Errorf("status=%d msg=%s", info.StatusCode, info.String())
}
return nil
}
func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.TaskRow, errCh chan<- error) error {
var (
err error
result *esapi.Response
ready = make(chan bool)
decoder *json.Decoder
hits = new(model.ESResponse)
)
if err = e.init(); err != nil {
logrus.Debugf("ES7.Start: init err=%v", err)
return err
}
qs := []func(*esapi.SearchRequest){
e.cli.Search.WithContext(util.TimeoutCtx(ctx, opt.ES7OperationTimeout)),
e.cli.Search.WithScroll(opt.ScrollTimeout),
e.cli.Search.WithSize(e.cfg.Size),
}
if e.cfg.Query != nil && len(e.cfg.Query) > 0 {
var bs []byte
if bs, err = json.Marshal(e.cfg.Query); err != nil {
logrus.Debugf("ES7.Start: marshal query err=%v", err)
return err
}
qs = append(qs, e.cli.Search.WithBody(bytes.NewReader(bs)))
}
go func() {
defer func() {
if e.scroll != "" {
var csr *esapi.Response
if csr, err = e.cli.ClearScroll(
e.cli.ClearScroll.WithContext(util.TimeoutCtx(ctx, 5)),
e.cli.ClearScroll.WithScrollID(e.scroll),
); err != nil {
logrus.Warnf("ES7.Start: clear scroll=%s err=%v", e.scroll, err)
} else {
if csr.StatusCode != 200 {
logrus.Warnf("ES7.Start: clear scroll=%s status=%d msg=%s", e.scroll, csr.StatusCode, csr.String())
}
}
}
close(rowCh)
close(errCh)
}()
ready <- true
if result, err = e.cli.Search(qs...); err != nil {
logrus.Debugf("ES7.Start: search err=%v", err)
errCh <- err
return
}
if err = util.CheckES7Response(result); err != nil {
logrus.Debugf("ES7.Start: search resp err=%v", err)
errCh <- err
return
}
decoder = json.NewDecoder(result.Body)
if err = decoder.Decode(hits); err != nil {
logrus.Debugf("ES7.Start: decode err=%v", err)
errCh <- err
return
}
if hits.TimedOut {
err = fmt.Errorf("timeout")
logrus.Debugf("ES7.Start: search timeout")
errCh <- err
return
}
e.scroll = hits.ScrollId
for idx := range hits.Hits.Hits {
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
return
}
for {
if result, err = e.cli.Scroll(
e.cli.Scroll.WithContext(util.TimeoutCtx(ctx, opt.ES7OperationTimeout)),
e.cli.Scroll.WithScrollID(e.scroll),
); err != nil {
logrus.Debugf("ES7.Start: search err=%v", err)
errCh <- err
return
}
if err = util.CheckES7Response(result); err != nil {
logrus.Debugf("ES7.Start: search resp err=%v", err)
errCh <- err
return
}
decoder = json.NewDecoder(result.Body)
hits = new(model.ESResponse)
if err = decoder.Decode(hits); err != nil {
logrus.Debugf("ES7.Start: decode err=%v", err)
errCh <- err
return
}
if hits.TimedOut {
err = fmt.Errorf("timeout")
logrus.Debugf("ES7.Start: search timeout")
errCh <- err
return
}
for idx := range hits.Hits.Hits {
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
return
}
}
}()
<-ready
return nil
}

45
internal/controller/output/xfile/local.go Normal file
View File

@ -0,0 +1,45 @@
package xfile
import (
"context"
"fmt"
"github.com/loveuer/nfflow/internal/model"
"io"
"os"
)
type LocalFile struct {
writer io.Writer
cfg struct {
MaxSize int
Path string
}
}
func (lf *LocalFile) init() error {
var (
err error
)
if _, err = os.Stat(lf.cfg.Path); !os.IsNotExist(err) {
return fmt.Errorf("file=%s already exist", lf.cfg.Path)
}
if lf.writer, err = os.OpenFile(lf.cfg.Path, os.O_CREATE|os.O_RDWR, 0644); err != nil {
return fmt.Errorf("openfile=%s err=%v", lf.cfg.Path, err)
}
return nil
}
func (lf *LocalFile) Start(ctx context.Context, rowCh <-chan *model.TaskRow, errCh chan<- error) error {
var (
err error
)
if err = lf.init(); err != nil {
return err
}
return nil
}

1
internal/controller/pipe/loadash/loadash.go Normal file
View File

@ -0,0 +1 @@
package loadash

52
internal/handler/task.go
View File

@ -9,6 +9,8 @@ import (
"github.com/loveuer/nfflow/internal/opt"
"github.com/loveuer/nfflow/internal/sqlType"
"github.com/loveuer/nfflow/internal/util"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"time"
)
@ -65,11 +67,11 @@ func TaskList(c *nf.Ctx) error {
func TaskCreate(c *nf.Ctx) error {
type Req struct {
TaskName string `json:"task_name"`
TaskType string `json:"task_type"`
Timeout int `json:"timeout"`
Cron string `json:"cron"`
RunAt int64 `json:"run_at"`
TaskName string `json:"task_name"`
TaskRunType model.TaskRunType `json:"task_run_type"`
TaskTimeout int `json:"task_timeout"`
TaskCron string `json:"task_cron"`
TaskRunAt int64 `json:"task_run_at"`
}
var (
@ -82,32 +84,46 @@ func TaskCreate(c *nf.Ctx) error {
return resp.Resp400(c, err.Error())
}
if req.TaskName != "" {
if req.TaskName == "" {
return resp.Resp400(c, req)
}
task := &model.Task{TaskName: req.TaskName}
if req.Timeout < opt.TaskMinTimeout || req.Timeout > opt.TaskMaxTimeout {
if req.TaskTimeout < opt.TaskMinTimeout || req.TaskTimeout > opt.TaskMaxTimeout {
return resp.Resp400(c, req, fmt.Sprintf("timeout 时长过短(%d - %d)", opt.TaskMinTimeout, opt.TaskMaxTimeout))
}
task.TimeoutSecond = req.Timeout
task.TaskTimeout = req.TaskTimeout
switch req.TaskType {
case "once":
case "timing":
rt := time.UnixMilli(req.RunAt)
if rt.Sub(now).Seconds() > opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务执行时间距离当前时间太短")
switch req.TaskRunType {
case model.TaskRunTypeOnce:
case model.TaskRunTypeTiming:
runAt := time.UnixMilli(req.TaskRunAt)
if runAt.Sub(now).Seconds() < 2*opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务定时时间太短")
}
task.TaskRunType = fmt.Sprintf("T-%d", req.RunAt)
case "cron":
task.TaskRunType = fmt.Sprintf("C-%s", req.TaskType)
task.TaskRunAt = req.TaskRunAt
case model.TaskRunTypeCron:
var schedule cron.Schedule
if schedule, err = opt.CronParser.Parse(req.TaskCron); err != nil {
return resp.Resp400(c, err.Error())
}
logrus.Debugf("TaskCreate: task cron=%s schedule next=%v", req.TaskCron, schedule.Next(now))
if schedule.Next(now).Sub(now).Seconds() < 2*opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务定时时间太短")
}
task.TaskCron = req.TaskCron
default:
return resp.Resp400(c, req, "任务执行类型: once/timing/cron")
return resp.Resp400(c, req, "错误的任务运行类型")
}
task.TaskRunType = req.TaskRunType
if err = database.DB.Session(util.Timeout(5)).
Create(task).
Error; err != nil {

27
internal/model/es.go Normal file
View File

@ -0,0 +1,27 @@
package model
type ESDoc struct {
DocId string `json:"_id"`
Index string `json:"_index"`
Content map[string]any `json:"_source"`
}
type ESResponse struct {
ScrollId string `json:"_scroll_id"`
Took int `json:"took"`
TimedOut bool `json:"timed_out"`
Shards struct {
Total int `json:"total"`
Successful int `json:"successful"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
} `json:"_shards"`
Hits struct {
Total struct {
Value int `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
MaxScore float64 `json:"max_score"`
Hits []*ESDoc `json:"hits"`
} `json:"hits"`
}

3
internal/model/interface.go
View File

@ -15,3 +15,6 @@ type OpLogger interface {
Render(content map[string]any) (string, error)
Template() string
}
type TaskRow interface {
}

58
internal/model/task.go
View File

@ -89,17 +89,65 @@ func (t TaskStatus) Label() string {
}
}
type TaskRunType int64
const (
TaskRunTypeOnce TaskRunType = iota
TaskRunTypeTiming
TaskRunTypeCron
)
func (t TaskRunType) Value() int64 { return int64(t) }
func (t TaskRunType) Code() string {
switch t {
case TaskRunTypeOnce:
return "once"
case TaskRunTypeTiming:
return "timing"
case TaskRunTypeCron:
return "cron"
default:
return "unknown"
}
}
func (t TaskRunType) Label() string {
switch t {
case TaskRunTypeOnce:
return "手动执行"
case TaskRunTypeTiming:
return "定时执行"
case TaskRunTypeCron:
return "周期执行"
default:
return "未知"
}
}
func (t TaskRunType) MarshalJSON() ([]byte, error) {
return json.Marshal(map[string]any{"code": t.Code(), "value": t.Value(), "label": t.Label()})
}
func (t TaskRunType) All() []Enum {
return []Enum{TaskRunTypeOnce, TaskRunTypeTiming, TaskRunTypeCron}
}
var _ Enum = TaskRunType(0)
type Task struct {
Id uint64 `json:"id" gorm:"primaryKey;column:id"`
CreatedAt int64 `json:"created_at" gorm:"column:created_at;autoCreateTime:milli"`
UpdatedAt int64 `json:"updated_at" gorm:"column:updated_at;autoUpdateTime:milli"`
DeletedAt int64 `json:"deleted_at" gorm:"index;column:deleted_at;default:0"`
TaskName string `json:"task_name" gorm:"column:task_name;type:varchar(256)"`
TaskRunType string `json:"task_run_type" gorm:"column:task_run_type;type:varchar(16);default:once"` // cron: C-"cron syntax", "once", timestamp: T-1234567890123 毫秒时间戳
TimeoutSecond int `json:"timeout_second" gorm:"column:timeout_second"`
TaskStatus TaskStatus `json:"task_status" gorm:"column:task_status"`
TaskLog string `json:"task_log" gorm:"task_log"`
TaskName string `json:"task_name" gorm:"column:task_name;type:varchar(256)"`
TaskRunType TaskRunType `json:"task_run_type" gorm:"column:task_run_type;type:varchar(16);default:0"` // cron: C-"cron syntax", "once", timestamp: T-1234567890123 毫秒时间戳
TaskRunAt int64 `json:"task_run_at" gorm:"column:task_run_at"`
TaskCron string `json:"task_cron" gorm:"column:task_cron;type:varchar(32)"`
TaskTimeout int `json:"task_timeout" gorm:"column:task_timeout"`
TaskStatus TaskStatus `json:"task_status" gorm:"column:task_status"`
TaskLog string `json:"task_log" gorm:"task_log"`
}
type Input struct {

10
internal/opt/var.go
View File

@ -1,6 +1,9 @@
package opt
import "time"
import (
"github.com/robfig/cron/v3"
"time"
)
const (
Version = "v0.0.1"
@ -38,9 +41,14 @@ const (
TaskMinTimeout = 10
TaskMaxTimeout = 24 * 3600
TaskFetchInterval = 5 * 60
ES7OperationTimeout = 30
)
var (
// todo: 颁发的 token, (cookie) 在缓存中存在的时间 (每次请求该时间也会被刷新)
TokenTimeout = time.Duration(3600*12) * time.Second
CronParser = cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
ScrollTimeout = time.Duration(5) * time.Minute
)

16
internal/util/ctx.go
View File

@ -20,3 +20,19 @@ func Timeout(seconds ...int) (ctx context.Context) {
return
}
func TimeoutCtx(parent context.Context, seconds ...int) (ctx context.Context) {
var (
duration time.Duration
)
if len(seconds) > 0 && seconds[0] > 0 {
duration = time.Duration(seconds[0]) * time.Second
} else {
duration = time.Duration(30) * time.Second
}
ctx, _ = context.WithTimeout(parent, duration)
return
}

14
internal/util/es7.go Normal file
View File

@ -0,0 +1,14 @@
package util
import (
"fmt"
"github.com/elastic/go-elasticsearch/v7/esapi"
)
func CheckES7Response(result *esapi.Response) error {
if result.StatusCode != 200 {
return fmt.Errorf("status=%s msg=%s", result.StatusCode, result.String())
}
return nil
}