441 lines
11 KiB
Go
441 lines
11 KiB
Go
package loadash
|
|
|
|
import (
|
|
"encoding/json"
|
|
"github.com/dop251/goja"
|
|
_ "github.com/dop251/goja"
|
|
"github.com/loveuer/nfflow/internal/model"
|
|
"github.com/robertkrimen/otto"
|
|
"testing"
|
|
)
|
|
|
|
func TestMapByOtto(t *testing.T) {
|
|
docs := []*model.ESDoc{
|
|
{DocId: "01", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "UTAustin",
|
|
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"input_date": 1683803316,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "02", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "3x926amfj3fxpki",
|
|
"id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
|
|
"input_date": 1683803324,
|
|
"platform": "kuaishou",
|
|
},
|
|
}},
|
|
{DocId: "03", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "tyler",
|
|
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"input_date": 1683821249,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "04", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "ErikVoorhees",
|
|
"id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"input_date": 1683821282,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "05", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "AbleEngineering",
|
|
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"input_date": 1683820912,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
}
|
|
|
|
myFunc := `var myFunc = function(item) { return {username: item._source._source.user_id} }`
|
|
|
|
vm := otto.New()
|
|
//if _, err = vm.Compile("babel", string(bfbs)); err != nil {
|
|
// t.Error("compile babel err:", err)
|
|
// return
|
|
//}
|
|
|
|
if _, err := vm.Run(myFunc); err != nil {
|
|
t.Error("1 err:", err)
|
|
return
|
|
}
|
|
|
|
for _, doc := range docs {
|
|
bs, _ := json.Marshal(doc)
|
|
val, err := otto.ToValue(string(bs))
|
|
if err != nil {
|
|
t.Error("to value err:", err)
|
|
return
|
|
}
|
|
|
|
vm.Set("doc", val)
|
|
|
|
_, err = vm.Run(`var result = myFunc(JSON.parse(doc))`)
|
|
if err != nil {
|
|
t.Error("2 err:", err)
|
|
return
|
|
}
|
|
|
|
value, err := vm.Get("result")
|
|
if err != nil {
|
|
t.Error("3 err:", err)
|
|
return
|
|
}
|
|
|
|
result, err := value.MarshalJSON()
|
|
if err != nil {
|
|
t.Error("4 err:", err)
|
|
}
|
|
|
|
t.Log("result: ", string(result))
|
|
}
|
|
}
|
|
|
|
func BenchmarkByOtto(b *testing.B) {
|
|
docs := []*model.ESDoc{
|
|
{DocId: "01", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "UTAustin",
|
|
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"input_date": 1683803316,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "02", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "3x926amfj3fxpki",
|
|
"id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
|
|
"input_date": 1683803324,
|
|
"platform": "kuaishou",
|
|
},
|
|
}},
|
|
{DocId: "03", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "tyler",
|
|
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"input_date": 1683821249,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "04", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "ErikVoorhees",
|
|
"id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"input_date": 1683821282,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{DocId: "05", Index: "sonar_post", Content: map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "AbleEngineering",
|
|
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"input_date": 1683820912,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
}
|
|
|
|
myFunc := `var myFunc = function(item) { return {username: item["_source"]["_source"]["user_id"]} }`
|
|
|
|
vm := otto.New()
|
|
|
|
if _, err := vm.Run(myFunc); err != nil {
|
|
b.Error("1 err:", err)
|
|
return
|
|
}
|
|
|
|
for n := 0; n < b.N; n++ {
|
|
bs, _ := json.Marshal(docs[n%5])
|
|
val, err := otto.ToValue(string(bs))
|
|
if err != nil {
|
|
b.Error("to value err:", err)
|
|
return
|
|
}
|
|
|
|
vm.Set("doc", val)
|
|
|
|
_, err = vm.Run(`var result = myFunc(JSON.parse(doc))`)
|
|
if err != nil {
|
|
b.Error("2 err:", err)
|
|
return
|
|
}
|
|
|
|
value, err := vm.Get("result")
|
|
if err != nil {
|
|
b.Error("3 err:", err)
|
|
return
|
|
}
|
|
|
|
result, err := value.MarshalJSON()
|
|
if err != nil {
|
|
b.Error("4 err:", err)
|
|
}
|
|
|
|
_ = result
|
|
}
|
|
}
|
|
|
|
func TestByGoJa(t *testing.T) {
|
|
docs := []map[string]any{
|
|
{"doc_id": "01", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "UTAustin",
|
|
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"input_date": 1683803316,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "02", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "3x926amfj3fxpki",
|
|
"id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
|
|
"input_date": 1683803324,
|
|
"platform": "kuaishou",
|
|
},
|
|
}},
|
|
{"doc_id": "03", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "tyler",
|
|
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"input_date": 1683821249,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "04", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "ErikVoorhees",
|
|
"id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"input_date": 1683821282,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "05", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "AbleEngineering",
|
|
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"input_date": 1683820912,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
}
|
|
|
|
myFunc := `let result; let doc; let myFn;`
|
|
|
|
vm := goja.New()
|
|
|
|
var (
|
|
err error
|
|
)
|
|
|
|
if _, err = vm.RunString(myFunc); err != nil {
|
|
t.Error("1 err:", err)
|
|
}
|
|
|
|
vm.RunString(`myFn = ` + `item => item.content._source.platform`)
|
|
|
|
for _, doc := range docs {
|
|
vm.Set("doc", doc)
|
|
|
|
t.Log("log doc: ", vm.Get("doc").Export())
|
|
|
|
_, err = vm.RunString(`result = myFn(doc)`)
|
|
if err != nil {
|
|
t.Error("2 err:", err)
|
|
return
|
|
}
|
|
|
|
value := vm.Get("result").Export()
|
|
|
|
//fm, _ := value.(map[string]any)
|
|
|
|
t.Logf("result: %+v", value)
|
|
}
|
|
}
|
|
|
|
func BenchmarkByGoJa(b *testing.B) {
|
|
docs := []map[string]any{
|
|
{"doc_id": "01", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "UTAustin",
|
|
"id": "0e2071290cf0c1d08bc211b2452cc1b4",
|
|
"input_date": 1683803316,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "02", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "3x926amfj3fxpki",
|
|
"id": "0d47e9eecfa42704efe6c054cca7050f",
|
|
"title": "五四青年节|重温入团誓词，焕发青春斗志！",
|
|
"input_date": 1683803324,
|
|
"platform": "kuaishou",
|
|
},
|
|
}},
|
|
{"doc_id": "03", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "tyler",
|
|
"id": "dbe1091f2b7f2888e2004b2d71f7c51a",
|
|
"input_date": 1683821249,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "04", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "ErikVoorhees",
|
|
"id": "77c68a46f85f7d72ff900abe7b86eee7",
|
|
"input_date": 1683821282,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
{"doc_id": "05", "index": "sonar_post", "content": map[string]any{
|
|
"_index": "sonar_post1",
|
|
"_type": "_doc",
|
|
"_id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"_score": 1,
|
|
"_source": map[string]any{
|
|
"user_id": "AbleEngineering",
|
|
"id": "38d4aa64e21bd9e20c597c8c10ad9371",
|
|
"input_date": 1683820912,
|
|
"platform": "twitter",
|
|
},
|
|
}},
|
|
}
|
|
|
|
myFunc := `let result; let doc; let myFn = item => item.content `
|
|
|
|
vm := goja.New()
|
|
|
|
var (
|
|
err error
|
|
)
|
|
|
|
if _, err = vm.RunString(myFunc); err != nil {
|
|
b.Error("1 err:", err)
|
|
return
|
|
}
|
|
|
|
for n := 0; n < b.N; n++ {
|
|
doc := docs[n%5]
|
|
if err = vm.Set("doc", doc); err != nil {
|
|
b.Error("set doc err:", err)
|
|
return
|
|
}
|
|
|
|
if _, err = vm.RunString(`result = myFn(doc)`); err != nil {
|
|
b.Error("dial myFn err:", err)
|
|
return
|
|
}
|
|
|
|
value := vm.Get("result")
|
|
|
|
result := value.Export()
|
|
|
|
_ = result
|
|
}
|
|
}
|
|
|
|
func TestNewPipeMap(t *testing.T) {
|
|
pipe, err := NewPipeMap(`
|
|
item => {
|
|
return {id: item.doc_id, name: "static name", title: item.title + item.content}
|
|
}
|
|
`)
|
|
if err != nil {
|
|
t.Error(1, err)
|
|
return
|
|
}
|
|
|
|
data, err := pipe.Pipe(map[string]any{"doc_id": "220113", "title": "zyp", "content": "hello world"})
|
|
if err != nil {
|
|
t.Error(2, err)
|
|
return
|
|
}
|
|
|
|
t.Logf("data: %+v", data)
|
|
}
|