nfflow/internal/controller/pipe/loadash/loadash_test.go

package loadash

import (
	"encoding/json"
	"fmt"
	"github.com/loveuer/nfflow/internal/model"
	"github.com/robertkrimen/otto"
	"os"
	"os/exec"
	"testing"

	_ "github.com/expr-lang/expr"
)

func TestMapByOtto(t *testing.T) {
	docs := []*model.ESDoc{
		{DocId: "01", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "0e2071290cf0c1d08bc211b2452cc1b4",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "UTAustin",
				"id":         "0e2071290cf0c1d08bc211b2452cc1b4",
				"input_date": 1683803316,
				"platform":   "twitter",
			},
		}},
		{DocId: "02", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "0d47e9eecfa42704efe6c054cca7050f",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "3x926amfj3fxpki",
				"id":         "0d47e9eecfa42704efe6c054cca7050f",
				"title":      "五四青年节|重温入团誓词，焕发青春斗志！",
				"input_date": 1683803324,
				"platform":   "kuaishou",
			},
		}},
		{DocId: "03", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "dbe1091f2b7f2888e2004b2d71f7c51a",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "tyler",
				"id":         "dbe1091f2b7f2888e2004b2d71f7c51a",
				"input_date": 1683821249,
				"platform":   "twitter",
			},
		}},
		{DocId: "04", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "77c68a46f85f7d72ff900abe7b86eee7",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "ErikVoorhees",
				"id":         "77c68a46f85f7d72ff900abe7b86eee7",
				"input_date": 1683821282,
				"platform":   "twitter",
			},
		}},
		{DocId: "05", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "38d4aa64e21bd9e20c597c8c10ad9371",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "AbleEngineering",
				"id":         "38d4aa64e21bd9e20c597c8c10ad9371",
				"input_date": 1683820912,
				"platform":   "twitter",
			},
		}},
	}

	myFunc := `function myFunc(item) { return item["_source"]["user_id"] }`

	vm := otto.New()
	if _, err := vm.Run(myFunc); err != nil {
		t.Error("1 err:", err)
		return
	}

	for _, doc := range docs {
		bs, _ := json.Marshal(doc)
		val, err := otto.ToValue(string(bs))
		if err != nil {
			t.Error("to value err:", err)
			return
		}

		vm.Set("doc", val)

		_, err = vm.Run(`var result = JSON.parse(doc)`)
		if err != nil {
			t.Error("2 err:", err)
			return
		}

		value, err := vm.Get("result")
		if err != nil {
			t.Error("3 err:", err)
			return
		}

		result, err := value.MarshalJSON()
		if err != nil {
			t.Error("4 err:", err)
		}

		t.Log("result: ", string(result))
	}
}

func TestMapByNode(t *testing.T) {
	docs := []*model.ESDoc{
		{DocId: "01", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "0e2071290cf0c1d08bc211b2452cc1b4",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "UTAustin",
				"id":         "0e2071290cf0c1d08bc211b2452cc1b4",
				"input_date": 1683803316,
				"platform":   "twitter",
			},
		}},
		{DocId: "02", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "0d47e9eecfa42704efe6c054cca7050f",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "3x926amfj3fxpki",
				"id":         "0d47e9eecfa42704efe6c054cca7050f",
				"title":      "五四青年节|重温入团誓词，焕发青春斗志！",
				"input_date": 1683803324,
				"platform":   "kuaishou",
			},
		}},
		{DocId: "03", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "dbe1091f2b7f2888e2004b2d71f7c51a",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "tyler",
				"id":         "dbe1091f2b7f2888e2004b2d71f7c51a",
				"input_date": 1683821249,
				"platform":   "twitter",
			},
		}},
		{DocId: "04", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "77c68a46f85f7d72ff900abe7b86eee7",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "ErikVoorhees",
				"id":         "77c68a46f85f7d72ff900abe7b86eee7",
				"input_date": 1683821282,
				"platform":   "twitter",
			},
		}},
		{DocId: "05", Index: "sonar_post", Content: map[string]any{
			"_index": "sonar_post1",
			"_type":  "_doc",
			"_id":    "38d4aa64e21bd9e20c597c8c10ad9371",
			"_score": 1,
			"_source": map[string]any{
				"user_id":    "AbleEngineering",
				"id":         "38d4aa64e21bd9e20c597c8c10ad9371",
				"input_date": 1683820912,
				"platform":   "twitter",
			},
		}},
	}
	myFunc := `const myMap = function(item) { return item["_source"]["user_id"] }; let result;`

	cmd := exec.Command("node")
	cmd.Stdin.Read([]byte(myFunc))
	cmd.Stdout = os.Stdout

	for _, doc := range docs {
		bs, _ := json.Marshal(doc)
		cmd.Stdin.Read([]byte(fmt.Sprintf(`result = myMap(JSON.parse(%s))`, string(bs))))
		cmd.Stdin.Read([]byte(`console.log(result)`))
	}
}