loveuer/nfflow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: 调研 runtime script support

Browse Source
This commit is contained in:
loveuer
2024-04-01 22:17:49 +08:00
parent 4c40041b3d
commit 99f0b85d28
3 changed files with 204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
internal/controller/pipe/loadash/loadash_test.go Normal file
View File

@ -0,0 +1,194 @@
package loadash
import (
"encoding/json"
"fmt"
"github.com/loveuer/nfflow/internal/model"
"github.com/robertkrimen/otto"
"os"
"os/exec"
"testing"
_ "github.com/expr-lang/expr"
)
func TestMapByOtto(t *testing.T) {
docs := []*model.ESDoc{
{DocId: "01", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
"_score": 1,
"_source": map[string]any{
"user_id": "UTAustin",
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
"input_date": 1683803316,
"platform": "twitter",
},
}},
{DocId: "02", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "0d47e9eecfa42704efe6c054cca7050f",
"_score": 1,
"_source": map[string]any{
"user_id": "3x926amfj3fxpki",
"id": "0d47e9eecfa42704efe6c054cca7050f",
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
"input_date": 1683803324,
"platform": "kuaishou",
},
}},
{DocId: "03", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
"_score": 1,
"_source": map[string]any{
"user_id": "tyler",
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
"input_date": 1683821249,
"platform": "twitter",
},
}},
{DocId: "04", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
"_score": 1,
"_source": map[string]any{
"user_id": "ErikVoorhees",
"id": "77c68a46f85f7d72ff900abe7b86eee7",
"input_date": 1683821282,
"platform": "twitter",
},
}},
{DocId: "05", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
"_score": 1,
"_source": map[string]any{
"user_id": "AbleEngineering",
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
"input_date": 1683820912,
"platform": "twitter",
},
}},
}
myFunc := `function myFunc(item) { return item["_source"]["user_id"] }`
vm := otto.New()
if _, err := vm.Run(myFunc); err != nil {
t.Error("1 err:", err)
return
}
for _, doc := range docs {
bs, _ := json.Marshal(doc)
val, err := otto.ToValue(string(bs))
if err != nil {
t.Error("to value err:", err)
return
}
vm.Set("doc", val)
_, err = vm.Run(`var result = JSON.parse(doc)`)
if err != nil {
t.Error("2 err:", err)
return
}
value, err := vm.Get("result")
if err != nil {
t.Error("3 err:", err)
return
}
result, err := value.MarshalJSON()
if err != nil {
t.Error("4 err:", err)
}
t.Log("result: ", string(result))
}
}
func TestMapByNode(t *testing.T) {
docs := []*model.ESDoc{
{DocId: "01", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
"_score": 1,
"_source": map[string]any{
"user_id": "UTAustin",
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
"input_date": 1683803316,
"platform": "twitter",
},
}},
{DocId: "02", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "0d47e9eecfa42704efe6c054cca7050f",
"_score": 1,
"_source": map[string]any{
"user_id": "3x926amfj3fxpki",
"id": "0d47e9eecfa42704efe6c054cca7050f",
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
"input_date": 1683803324,
"platform": "kuaishou",
},
}},
{DocId: "03", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
"_score": 1,
"_source": map[string]any{
"user_id": "tyler",
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
"input_date": 1683821249,
"platform": "twitter",
},
}},
{DocId: "04", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
"_score": 1,
"_source": map[string]any{
"user_id": "ErikVoorhees",
"id": "77c68a46f85f7d72ff900abe7b86eee7",
"input_date": 1683821282,
"platform": "twitter",
},
}},
{DocId: "05", Index: "sonar_post", Content: map[string]any{
"_index": "sonar_post1",
"_type": "_doc",
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
"_score": 1,
"_source": map[string]any{
"user_id": "AbleEngineering",
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
"input_date": 1683820912,
"platform": "twitter",
},
}},
}
myFunc := `const myMap = function(item) { return item["_source"]["user_id"] }; let result;`
cmd := exec.Command("node")
cmd.Stdin.Read([]byte(myFunc))
cmd.Stdout = os.Stdout
for _, doc := range docs {
bs, _ := json.Marshal(doc)
cmd.Stdin.Read([]byte(fmt.Sprintf(`result = myMap(JSON.parse(%s))`, string(bs))))
cmd.Stdin.Read([]byte(`console.log(result)`))
}
}