loveuer/nfflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: output, pipe

Browse Source
This commit is contained in:
loveuer 2024-04-01 18:13:10 +08:00
parent 195fbcd308
commit 4c40041b3d
16 changed files with 445 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
front/src/app/page/task/task.component.html
View File

@ -16,6 +16,16 @@
<td mat-cell *matCellDef="let element"> {{ element.task_name }}</td>
</ng-container>
<ng-container matColumnDef="updated_at">
<th mat-header-cell *matHeaderCellDef>updated_at</th>
<td mat-cell *matCellDef="let element"> {{ element.updated_at | date: "yyyy-MM-dd HH:mm:SS" }}</td>
</ng-container>
<ng-container matColumnDef="task_run_type">
<th mat-header-cell *matHeaderCellDef>task_run_type</th>
<td mat-cell *matCellDef="let element" [matTooltip]="element.task_cron"> {{ element.task_run_type.label }}</td>
</ng-container>
<ng-container matColumnDef="task_status">
<th mat-header-cell *matHeaderCellDef>task_status</th>
<td mat-cell *matCellDef="let element"> {{ element.task_status.label }}</td>

6
front/src/app/page/task/task.component.ts
View File

@ -24,18 +24,18 @@ import {MatTooltip} from "@angular/material/tooltip";
styleUrl: './task.component.scss'
})
export class TaskComponent {
displayedColumns = ["id", "task_name", "task_status", "operation"];
displayedColumns = ["id", "task_name","updated_at", "task_run_type", "task_status", "operation"];
constructor(
public task_srv: TaskService,
) {
}
update_task(element:Task) {
update_task(element: Task) {
}
delete_task(element:Task) {
delete_task(element: Task) {
}

1
go.mod
View File

@ -10,6 +10,7 @@ require (
github.com/golang-jwt/jwt/v5 v5.2.0
github.com/jackc/pgtype v1.12.0
github.com/loveuer/nf v0.1.3
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.39.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.6.0

2
go.sum
View File

@ -127,6 +127,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=

191
internal/controller/input/es7/es7.go Normal file
View File

@ -0,0 +1,191 @@
package es7
import (
"bytes"
"context"
"encoding/json"
"fmt"
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/loveuer/nfflow/internal/model"
"github.com/loveuer/nfflow/internal/opt"
"github.com/loveuer/nfflow/internal/util"
"github.com/sirupsen/logrus"
)
type ES7 struct {
cli *elastic.Client
scroll string
cfg struct {
Endpoints []string
Username string
Password string
Size int
Query map[string]any
Source []string
}
}
func (e *ES7) init() error {
var (
err error
cfg = elastic.Config{
Addresses: e.cfg.Endpoints,
Username: e.cfg.Username,
Password: e.cfg.Password,
RetryOnStatus: []int{429},
}
info *esapi.Response
)
if e.cli, err = elastic.NewClient(cfg); err != nil {
return err
}
if info, err = e.cli.Info(e.cli.Info.WithContext(util.Timeout(5))); err != nil {
return err
}
if info.StatusCode != 200 {
return fmt.Errorf("status=%d msg=%s", info.StatusCode, info.String())
}
return nil
}
func (e *ES7) Start(ctx context.Context, task *model.Task, rowCh chan<- model.TaskRow, errCh chan<- error) error {
var (
err error
result *esapi.Response
ready = make(chan bool)
decoder *json.Decoder
hits = new(model.ESResponse)
)
if err = e.init(); err != nil {
logrus.Debugf("ES7.Start: init err=%v", err)
return err
}
qs := []func(*esapi.SearchRequest){
e.cli.Search.WithContext(util.TimeoutCtx(ctx, opt.ES7OperationTimeout)),
e.cli.Search.WithScroll(opt.ScrollTimeout),
e.cli.Search.WithSize(e.cfg.Size),
}
if e.cfg.Query != nil && len(e.cfg.Query) > 0 {
var bs []byte
if bs, err = json.Marshal(e.cfg.Query); err != nil {
logrus.Debugf("ES7.Start: marshal query err=%v", err)
return err
}
qs = append(qs, e.cli.Search.WithBody(bytes.NewReader(bs)))
}
go func() {
defer func() {
if e.scroll != "" {
var csr *esapi.Response
if csr, err = e.cli.ClearScroll(
e.cli.ClearScroll.WithContext(util.TimeoutCtx(ctx, 5)),
e.cli.ClearScroll.WithScrollID(e.scroll),
); err != nil {
logrus.Warnf("ES7.Start: clear scroll=%s err=%v", e.scroll, err)
} else {
if csr.StatusCode != 200 {
logrus.Warnf("ES7.Start: clear scroll=%s status=%d msg=%s", e.scroll, csr.StatusCode, csr.String())
}
}
}
close(rowCh)
close(errCh)
}()
ready <- true
if result, err = e.cli.Search(qs...); err != nil {
logrus.Debugf("ES7.Start: search err=%v", err)
errCh <- err
return
}
if err = util.CheckES7Response(result); err != nil {
logrus.Debugf("ES7.Start: search resp err=%v", err)
errCh <- err
return
}
decoder = json.NewDecoder(result.Body)
if err = decoder.Decode(hits); err != nil {
logrus.Debugf("ES7.Start: decode err=%v", err)
errCh <- err
return
}
if hits.TimedOut {
err = fmt.Errorf("timeout")
logrus.Debugf("ES7.Start: search timeout")
errCh <- err
return
}
e.scroll = hits.ScrollId
for idx := range hits.Hits.Hits {
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
return
}
for {
if result, err = e.cli.Scroll(
e.cli.Scroll.WithContext(util.TimeoutCtx(ctx, opt.ES7OperationTimeout)),
e.cli.Scroll.WithScrollID(e.scroll),
); err != nil {
logrus.Debugf("ES7.Start: search err=%v", err)
errCh <- err
return
}
if err = util.CheckES7Response(result); err != nil {
logrus.Debugf("ES7.Start: search resp err=%v", err)
errCh <- err
return
}
decoder = json.NewDecoder(result.Body)
hits = new(model.ESResponse)
if err = decoder.Decode(hits); err != nil {
logrus.Debugf("ES7.Start: decode err=%v", err)
errCh <- err
return
}
if hits.TimedOut {
err = fmt.Errorf("timeout")
logrus.Debugf("ES7.Start: search timeout")
errCh <- err
return
}
for idx := range hits.Hits.Hits {
rowCh <- hits.Hits.Hits[idx]
}
if len(hits.Hits.Hits) < e.cfg.Size {
return
}
}
}()
<-ready
return nil
}

45
internal/controller/output/xfile/local.go Normal file
View File

@ -0,0 +1,45 @@
package xfile
import (
"context"
"fmt"
"github.com/loveuer/nfflow/internal/model"
"io"
"os"
)
type LocalFile struct {
writer io.Writer
cfg struct {
MaxSize int
Path string
}
}
func (lf *LocalFile) init() error {
var (
err error
)
if _, err = os.Stat(lf.cfg.Path); !os.IsNotExist(err) {
return fmt.Errorf("file=%s already exist", lf.cfg.Path)
}
if lf.writer, err = os.OpenFile(lf.cfg.Path, os.O_CREATE|os.O_RDWR, 0644); err != nil {
return fmt.Errorf("openfile=%s err=%v", lf.cfg.Path, err)
}
return nil
}
func (lf *LocalFile) Start(ctx context.Context, rowCh <-chan *model.TaskRow, errCh chan<- error) error {
var (
err error
)
if err = lf.init(); err != nil {
return err
}
return nil
}

1
internal/controller/pipe/loadash/loadash.go Normal file
View File

@ -0,0 +1 @@
package loadash

50
internal/handler/task.go
View File

@ -9,6 +9,8 @@ import (
"github.com/loveuer/nfflow/internal/opt"
"github.com/loveuer/nfflow/internal/sqlType"
"github.com/loveuer/nfflow/internal/util"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"time"
)
@ -66,10 +68,10 @@ func TaskList(c *nf.Ctx) error {
func TaskCreate(c *nf.Ctx) error {
type Req struct {
TaskName string `json:"task_name"`
TaskType string `json:"task_type"`
Timeout int `json:"timeout"`
Cron string `json:"cron"`
RunAt int64 `json:"run_at"`
TaskRunType model.TaskRunType `json:"task_run_type"`
TaskTimeout int `json:"task_timeout"`
TaskCron string `json:"task_cron"`
TaskRunAt int64 `json:"task_run_at"`
}
var (
@ -82,32 +84,46 @@ func TaskCreate(c *nf.Ctx) error {
return resp.Resp400(c, err.Error())
}
if req.TaskName != "" {
if req.TaskName == "" {
return resp.Resp400(c, req)
}
task := &model.Task{TaskName: req.TaskName}
if req.Timeout < opt.TaskMinTimeout || req.Timeout > opt.TaskMaxTimeout {
if req.TaskTimeout < opt.TaskMinTimeout || req.TaskTimeout > opt.TaskMaxTimeout {
return resp.Resp400(c, req, fmt.Sprintf("timeout 时长过短(%d - %d)", opt.TaskMinTimeout, opt.TaskMaxTimeout))
}
task.TimeoutSecond = req.Timeout
task.TaskTimeout = req.TaskTimeout
switch req.TaskType {
case "once":
case "timing":
rt := time.UnixMilli(req.RunAt)
if rt.Sub(now).Seconds() > opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务执行时间距离当前时间太短")
switch req.TaskRunType {
case model.TaskRunTypeOnce:
case model.TaskRunTypeTiming:
runAt := time.UnixMilli(req.TaskRunAt)
if runAt.Sub(now).Seconds() < 2*opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务定时时间太短")
}
task.TaskRunType = fmt.Sprintf("T-%d", req.RunAt)
case "cron":
task.TaskRunType = fmt.Sprintf("C-%s", req.TaskType)
task.TaskRunAt = req.TaskRunAt
case model.TaskRunTypeCron:
var schedule cron.Schedule
if schedule, err = opt.CronParser.Parse(req.TaskCron); err != nil {
return resp.Resp400(c, err.Error())
}
logrus.Debugf("TaskCreate: task cron=%s schedule next=%v", req.TaskCron, schedule.Next(now))
if schedule.Next(now).Sub(now).Seconds() < 2*opt.TaskFetchInterval {
return resp.Resp400(c, req, "任务定时时间太短")
}
task.TaskCron = req.TaskCron
default:
return resp.Resp400(c, req, "任务执行类型: once/timing/cron")
return resp.Resp400(c, req, "错误的任务运行类型")
}
task.TaskRunType = req.TaskRunType
if err = database.DB.Session(util.Timeout(5)).
Create(task).
Error; err != nil {

27
internal/model/es.go Normal file
View File

@ -0,0 +1,27 @@
package model
type ESDoc struct {
DocId string `json:"_id"`
Index string `json:"_index"`
Content map[string]any `json:"_source"`
}
type ESResponse struct {
ScrollId string `json:"_scroll_id"`
Took int `json:"took"`
TimedOut bool `json:"timed_out"`
Shards struct {
Total int `json:"total"`
Successful int `json:"successful"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
} `json:"_shards"`
Hits struct {
Total struct {
Value int `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
MaxScore float64 `json:"max_score"`
Hits []*ESDoc `json:"hits"`
} `json:"hits"`
}

3
internal/model/interface.go
View File

@ -15,3 +15,6 @@ type OpLogger interface {
Render(content map[string]any) (string, error)
Template() string
}
type TaskRow interface {
}

52
internal/model/task.go
View File

@ -89,6 +89,52 @@ func (t TaskStatus) Label() string {
}
}
type TaskRunType int64
const (
TaskRunTypeOnce TaskRunType = iota
TaskRunTypeTiming
TaskRunTypeCron
)
func (t TaskRunType) Value() int64 { return int64(t) }
func (t TaskRunType) Code() string {
switch t {
case TaskRunTypeOnce:
return "once"
case TaskRunTypeTiming:
return "timing"
case TaskRunTypeCron:
return "cron"
default:
return "unknown"
}
}
func (t TaskRunType) Label() string {
switch t {
case TaskRunTypeOnce:
return "手动执行"
case TaskRunTypeTiming:
return "定时执行"
case TaskRunTypeCron:
return "周期执行"
default:
return "未知"
}
}
func (t TaskRunType) MarshalJSON() ([]byte, error) {
return json.Marshal(map[string]any{"code": t.Code(), "value": t.Value(), "label": t.Label()})
}
func (t TaskRunType) All() []Enum {
return []Enum{TaskRunTypeOnce, TaskRunTypeTiming, TaskRunTypeCron}
}
var _ Enum = TaskRunType(0)
type Task struct {
Id uint64 `json:"id" gorm:"primaryKey;column:id"`
CreatedAt int64 `json:"created_at" gorm:"column:created_at;autoCreateTime:milli"`
@ -96,8 +142,10 @@ type Task struct {
DeletedAt int64 `json:"deleted_at" gorm:"index;column:deleted_at;default:0"`
TaskName string `json:"task_name" gorm:"column:task_name;type:varchar(256)"`
TaskRunType string `json:"task_run_type" gorm:"column:task_run_type;type:varchar(16);default:once"` // cron: C-"cron syntax", "once", timestamp: T-1234567890123 毫秒时间戳
TimeoutSecond int `json:"timeout_second" gorm:"column:timeout_second"`
TaskRunType TaskRunType `json:"task_run_type" gorm:"column:task_run_type;type:varchar(16);default:0"` // cron: C-"cron syntax", "once", timestamp: T-1234567890123 毫秒时间戳
TaskRunAt int64 `json:"task_run_at" gorm:"column:task_run_at"`
TaskCron string `json:"task_cron" gorm:"column:task_cron;type:varchar(32)"`
TaskTimeout int `json:"task_timeout" gorm:"column:task_timeout"`
TaskStatus TaskStatus `json:"task_status" gorm:"column:task_status"`
TaskLog string `json:"task_log" gorm:"task_log"`
}

10
internal/opt/var.go
View File

@ -1,6 +1,9 @@
package opt
import "time"
import (
"github.com/robfig/cron/v3"
"time"
)
const (
Version = "v0.0.1"
@ -38,9 +41,14 @@ const (
TaskMinTimeout = 10
TaskMaxTimeout = 24 * 3600
TaskFetchInterval = 5 * 60
ES7OperationTimeout = 30
)
var (
// todo: 颁发的 token, (cookie) 在缓存中存在的时间 (每次请求该时间也会被刷新)
TokenTimeout = time.Duration(3600*12) * time.Second
CronParser = cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
ScrollTimeout = time.Duration(5) * time.Minute
)

16
internal/util/ctx.go
View File

@ -20,3 +20,19 @@ func Timeout(seconds ...int) (ctx context.Context) {
return
}
func TimeoutCtx(parent context.Context, seconds ...int) (ctx context.Context) {
var (
duration time.Duration
)
if len(seconds) > 0 && seconds[0] > 0 {
duration = time.Duration(seconds[0]) * time.Second
} else {
duration = time.Duration(30) * time.Second
}
ctx, _ = context.WithTimeout(parent, duration)
return
}

14
internal/util/es7.go Normal file
View File

@ -0,0 +1,14 @@
package util
import (
"fmt"
"github.com/elastic/go-elasticsearch/v7/esapi"
)
func CheckES7Response(result *esapi.Response) error {
if result.StatusCode != 200 {
return fmt.Errorf("status=%s msg=%s", result.StatusCode, result.String())
}
return nil
}

36
readme.md
View File

@ -1,35 +1,11 @@
# utl-one: utl all in one
# NF Flow
### Usage
### DEV
- 1. `git clone -b master --depth 1 http://10.220.10.35/dev/template/ultone.git {your_project_name}`
- 1. start service
- 2. `cd {your_project_name} && rm -rf .git && git init`
`go run .`
- 3. `go mod tidy`
- 2. start front
### Setting
#### 仔细查看项目中的 todo
#### 仔细查看 opt.var 中的设置
#### SQL
- sqlite:
- postgresql:
- mysql
#### Cache
- redis
- memory
### Feature
- 用户全功能模块
- 操作日志
### Next
- common user list (比如操作日志用户下拉)
`cd front && npm i && npm run start`

28
xhttp/task.http Normal file
View File

@ -0,0 +1,28 @@
### login
POST localhost/api/user/auth/login
Content-Type: application/json
{
"username": "admin",
"password": "Foobar123"
}
### create task 1
POST localhost/api/task/create
Content-Type: application/json
{
"task_name": "123",
"task_timeout": 3600
}
### create task 3
POST localhost/api/task/create
Content-Type: application/json
{
"task_name": "cron_123",
"task_timeout": 1800,
"task_run_type": 2,
"task_cron": "5 0 * * *"
}