package loadash import ( "encoding/json" "github.com/dop251/goja" _ "github.com/dop251/goja" "github.com/loveuer/nfflow/internal/model" "github.com/robertkrimen/otto" "testing" ) func TestMapByOtto(t *testing.T) { docs := []*model.ESDoc{ {DocId: "01", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0e2071290cf0c1d08bc211b2452cc1b4", "_score": 1, "_source": map[string]any{ "user_id": "UTAustin", "id": "0e2071290cf0c1d08bc211b2452cc1b4", "input_date": 1683803316, "platform": "twitter", }, }}, {DocId: "02", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0d47e9eecfa42704efe6c054cca7050f", "_score": 1, "_source": map[string]any{ "user_id": "3x926amfj3fxpki", "id": "0d47e9eecfa42704efe6c054cca7050f", "title": "五四青年节|重温入团誓词,焕发青春斗志!", "input_date": 1683803324, "platform": "kuaishou", }, }}, {DocId: "03", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "dbe1091f2b7f2888e2004b2d71f7c51a", "_score": 1, "_source": map[string]any{ "user_id": "tyler", "id": "dbe1091f2b7f2888e2004b2d71f7c51a", "input_date": 1683821249, "platform": "twitter", }, }}, {DocId: "04", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "77c68a46f85f7d72ff900abe7b86eee7", "_score": 1, "_source": map[string]any{ "user_id": "ErikVoorhees", "id": "77c68a46f85f7d72ff900abe7b86eee7", "input_date": 1683821282, "platform": "twitter", }, }}, {DocId: "05", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "38d4aa64e21bd9e20c597c8c10ad9371", "_score": 1, "_source": map[string]any{ "user_id": "AbleEngineering", "id": "38d4aa64e21bd9e20c597c8c10ad9371", "input_date": 1683820912, "platform": "twitter", }, }}, } myFunc := `var myFunc = function(item) { return {username: item._source._source.user_id} }` vm := otto.New() //if _, err = vm.Compile("babel", string(bfbs)); err != nil { // t.Error("compile babel err:", err) // return //} if _, err := vm.Run(myFunc); err != nil { t.Error("1 err:", err) return } for _, doc := range docs { bs, _ := json.Marshal(doc) val, err := otto.ToValue(string(bs)) if err != nil { t.Error("to value err:", err) return } vm.Set("doc", val) _, err = vm.Run(`var result = myFunc(JSON.parse(doc))`) if err != nil { t.Error("2 err:", err) return } value, err := vm.Get("result") if err != nil { t.Error("3 err:", err) return } result, err := value.MarshalJSON() if err != nil { t.Error("4 err:", err) } t.Log("result: ", string(result)) } } func BenchmarkByOtto(b *testing.B) { docs := []*model.ESDoc{ {DocId: "01", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0e2071290cf0c1d08bc211b2452cc1b4", "_score": 1, "_source": map[string]any{ "user_id": "UTAustin", "id": "0e2071290cf0c1d08bc211b2452cc1b4", "input_date": 1683803316, "platform": "twitter", }, }}, {DocId: "02", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0d47e9eecfa42704efe6c054cca7050f", "_score": 1, "_source": map[string]any{ "user_id": "3x926amfj3fxpki", "id": "0d47e9eecfa42704efe6c054cca7050f", "title": "五四青年节|重温入团誓词,焕发青春斗志!", "input_date": 1683803324, "platform": "kuaishou", }, }}, {DocId: "03", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "dbe1091f2b7f2888e2004b2d71f7c51a", "_score": 1, "_source": map[string]any{ "user_id": "tyler", "id": "dbe1091f2b7f2888e2004b2d71f7c51a", "input_date": 1683821249, "platform": "twitter", }, }}, {DocId: "04", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "77c68a46f85f7d72ff900abe7b86eee7", "_score": 1, "_source": map[string]any{ "user_id": "ErikVoorhees", "id": "77c68a46f85f7d72ff900abe7b86eee7", "input_date": 1683821282, "platform": "twitter", }, }}, {DocId: "05", Index: "sonar_post", Content: map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "38d4aa64e21bd9e20c597c8c10ad9371", "_score": 1, "_source": map[string]any{ "user_id": "AbleEngineering", "id": "38d4aa64e21bd9e20c597c8c10ad9371", "input_date": 1683820912, "platform": "twitter", }, }}, } myFunc := `var myFunc = function(item) { return {username: item["_source"]["_source"]["user_id"]} }` vm := otto.New() if _, err := vm.Run(myFunc); err != nil { b.Error("1 err:", err) return } for n := 0; n < b.N; n++ { bs, _ := json.Marshal(docs[n%5]) val, err := otto.ToValue(string(bs)) if err != nil { b.Error("to value err:", err) return } vm.Set("doc", val) _, err = vm.Run(`var result = myFunc(JSON.parse(doc))`) if err != nil { b.Error("2 err:", err) return } value, err := vm.Get("result") if err != nil { b.Error("3 err:", err) return } result, err := value.MarshalJSON() if err != nil { b.Error("4 err:", err) } _ = result } } func TestByGoJa(t *testing.T) { docs := []map[string]any{ {"doc_id": "01", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0e2071290cf0c1d08bc211b2452cc1b4", "_score": 1, "_source": map[string]any{ "user_id": "UTAustin", "id": "0e2071290cf0c1d08bc211b2452cc1b4", "input_date": 1683803316, "platform": "twitter", }, }}, {"doc_id": "02", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0d47e9eecfa42704efe6c054cca7050f", "_score": 1, "_source": map[string]any{ "user_id": "3x926amfj3fxpki", "id": "0d47e9eecfa42704efe6c054cca7050f", "title": "五四青年节|重温入团誓词,焕发青春斗志!", "input_date": 1683803324, "platform": "kuaishou", }, }}, {"doc_id": "03", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "dbe1091f2b7f2888e2004b2d71f7c51a", "_score": 1, "_source": map[string]any{ "user_id": "tyler", "id": "dbe1091f2b7f2888e2004b2d71f7c51a", "input_date": 1683821249, "platform": "twitter", }, }}, {"doc_id": "04", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "77c68a46f85f7d72ff900abe7b86eee7", "_score": 1, "_source": map[string]any{ "user_id": "ErikVoorhees", "id": "77c68a46f85f7d72ff900abe7b86eee7", "input_date": 1683821282, "platform": "twitter", }, }}, {"doc_id": "05", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "38d4aa64e21bd9e20c597c8c10ad9371", "_score": 1, "_source": map[string]any{ "user_id": "AbleEngineering", "id": "38d4aa64e21bd9e20c597c8c10ad9371", "input_date": 1683820912, "platform": "twitter", }, }}, } myFunc := `let result; let doc; let myFn;` vm := goja.New() var ( err error ) if _, err = vm.RunString(myFunc); err != nil { t.Error("1 err:", err) } vm.RunString(`myFn = ` + `item => item.content._source.platform`) for _, doc := range docs { vm.Set("doc", doc) t.Log("log doc: ", vm.Get("doc").Export()) _, err = vm.RunString(`result = myFn(doc)`) if err != nil { t.Error("2 err:", err) return } value := vm.Get("result").Export() //fm, _ := value.(map[string]any) t.Logf("result: %+v", value) } } func BenchmarkByGoJa(b *testing.B) { docs := []map[string]any{ {"doc_id": "01", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0e2071290cf0c1d08bc211b2452cc1b4", "_score": 1, "_source": map[string]any{ "user_id": "UTAustin", "id": "0e2071290cf0c1d08bc211b2452cc1b4", "input_date": 1683803316, "platform": "twitter", }, }}, {"doc_id": "02", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "0d47e9eecfa42704efe6c054cca7050f", "_score": 1, "_source": map[string]any{ "user_id": "3x926amfj3fxpki", "id": "0d47e9eecfa42704efe6c054cca7050f", "title": "五四青年节|重温入团誓词,焕发青春斗志!", "input_date": 1683803324, "platform": "kuaishou", }, }}, {"doc_id": "03", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "dbe1091f2b7f2888e2004b2d71f7c51a", "_score": 1, "_source": map[string]any{ "user_id": "tyler", "id": "dbe1091f2b7f2888e2004b2d71f7c51a", "input_date": 1683821249, "platform": "twitter", }, }}, {"doc_id": "04", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "77c68a46f85f7d72ff900abe7b86eee7", "_score": 1, "_source": map[string]any{ "user_id": "ErikVoorhees", "id": "77c68a46f85f7d72ff900abe7b86eee7", "input_date": 1683821282, "platform": "twitter", }, }}, {"doc_id": "05", "index": "sonar_post", "content": map[string]any{ "_index": "sonar_post1", "_type": "_doc", "_id": "38d4aa64e21bd9e20c597c8c10ad9371", "_score": 1, "_source": map[string]any{ "user_id": "AbleEngineering", "id": "38d4aa64e21bd9e20c597c8c10ad9371", "input_date": 1683820912, "platform": "twitter", }, }}, } myFunc := `let result; let doc; let myFn = item => item.content ` vm := goja.New() var ( err error ) if _, err = vm.RunString(myFunc); err != nil { b.Error("1 err:", err) return } for n := 0; n < b.N; n++ { doc := docs[n%5] if err = vm.Set("doc", doc); err != nil { b.Error("set doc err:", err) return } if _, err = vm.RunString(`result = myFn(doc)`); err != nil { b.Error("dial myFn err:", err) return } value := vm.Get("result") result := value.Export() _ = result } } func TestNewPipeMap(t *testing.T) { pipe, err := NewPipeMap(` item => { return {id: item.doc_id, name: "static name", title: item.title + item.content} } `) if err != nil { t.Error(1, err) return } data, err := pipe.Pipe(map[string]any{"doc_id": "220113", "title": "zyp", "content": "hello world"}) if err != nil { t.Error(2, err) return } t.Logf("data: %+v", data) }