fix: update user nickname;

feat: add database/kafka.

internal/database/db/client.go

@@ -3,25 +3,32 @@ package db
 import (
 	"context"
 	"io"
+
 	"ultone/internal/opt"
 	"ultone/internal/tool"
 
 	"gorm.io/gorm"
 )
 
-var (
-	Default *Client
+var Default *Client
+
+type DBType string
+
+const (
+	DBTypeSqlite   = "sqlite"
+	DBTypeMysql    = "mysql"
+	DBTypePostgres = "postgres"
 )
 
 type Client struct {
 	ctx       context.Context
 	cli       *gorm.DB
-	ttype     string
+	dbType    DBType
 	cfgSqlite *cfgSqlite
 }
 
-func (c *Client) Type() string {
-	return c.ttype
+func (c *Client) Type() DBType {
+	return c.dbType
 }
 
 func (c *Client) Session(ctxs ...context.Context) *gorm.DB {
@@ -49,7 +56,7 @@ func (c *Client) Close() {
 // Dump
 // Only for sqlite with mem mode to dump data to bytes(io.Reader)
 func (c *Client) Dump() (reader io.ReadSeekCloser, ok bool) {
-	if c.ttype != "sqlite" {
+	if c.dbType != DBTypeSqlite {
 		return nil, false
 	}
 

internal/database/db/db_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestOpen(t *testing.T) {
-	myClient, err := New(context.TODO(), "sqlite::", OptSqliteByMem())
+	myClient, err := New(context.TODO(), "sqlite::", OptSqliteByMem(nil))
 	if err != nil {
 		t.Fatalf("TestOpen: New err = %v", err)
 	}
@@ -37,7 +37,7 @@ func TestOpen(t *testing.T) {
 			t.Fatalf("TestOpen: ReadAll err = %v", err)
 		}
 
-		os.WriteFile("dump.db", bs, 0644)
+		os.WriteFile("dump.db", bs, 0o644)
 	}
 
 	myClient.Close()

internal/database/db/new.go

@@ -11,35 +11,38 @@ import (
 )
 
 func New(ctx context.Context, uri string, opts ...Option) (*Client, error) {
-	strs := strings.Split(uri, "::")
+	parts := strings.SplitN(uri, "::", 2)
 
-	if len(strs) != 2 {
+	if len(parts) != 2 {
 		return nil, fmt.Errorf("db.Init: opt db uri invalid: %s", uri)
 	}
 
-	c := &Client{ttype: strs[0], cfgSqlite: &cfgSqlite{fsType: "file"}}
+	c := &Client{cfgSqlite: &cfgSqlite{fsType: "file"}}
 	for _, f := range opts {
 		f(c)
 	}
 
 	var (
 		err error
-		dsn = strs[1]
+		dsn = parts[1]
 	)
 
-	switch strs[0] {
+	switch parts[0] {
 	case "sqlite":
+		c.dbType = DBTypeSqlite
 		err = openSqlite(c, dsn)
 	case "mysql":
+		c.dbType = DBTypeMysql
 		c.cli, err = gorm.Open(mysql.Open(dsn))
 	case "postgres":
+		c.dbType = DBTypePostgres
 		c.cli, err = gorm.Open(postgres.Open(dsn))
 	default:
-		return nil, fmt.Errorf("db type only support: [sqlite, mysql, postgres], unsupported db type: %s", strs[0])
+		return nil, fmt.Errorf("db type only support: [sqlite, mysql, postgres], unsupported db type: %s", parts[0])
 	}
 
 	if err != nil {
-		return nil, fmt.Errorf("db.Init: open %s with dsn:%s, err: %w", strs[0], dsn, err)
+		return nil, fmt.Errorf("db.Init: open %s with dsn:%s, err: %w", parts[0], dsn, err)
 	}
 
 	return c, nil

internal/database/kafka/client.go

@@ -0,0 +1,74 @@
+package kafka
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"ultone/internal/interfaces"
+
+	"github.com/loveuer/nf/nft/log"
+	kfkgo "github.com/segmentio/kafka-go"
+	"github.com/segmentio/kafka-go/sasl"
+)
+
+var Client *client
+
+type client struct {
+	sync.Mutex
+	ctx          context.Context
+	d            *kfkgo.Dialer
+	topic        string
+	partition    int
+	reconnection bool
+	mechanism    sasl.Mechanism
+	address      string
+	logger       interfaces.Logger
+	writer       *kfkgo.Writer
+}
+
+func Init(address string, opts ...OptionFn) error {
+	c, err := New(address, opts...)
+	if err != nil {
+		return err
+	}
+
+	Client = c
+
+	return nil
+}
+
+func New(address string, opts ...OptionFn) (*client, error) {
+	c := &client{}
+
+	if address == "" {
+		return nil, fmt.Errorf("address required")
+	}
+
+	for _, fn := range opts {
+		fn(c)
+	}
+
+	c.address = address
+
+	if c.ctx == nil {
+		c.ctx = context.Background()
+	}
+
+	if c.logger == nil {
+		c.logger = log.New()
+	}
+
+	dia := &kfkgo.Dialer{
+		Timeout: 30 * time.Second,
+	}
+
+	if c.mechanism != nil {
+		dia.SASLMechanism = c.mechanism
+	}
+
+	c.d = dia
+
+	return c, nil
+}

internal/database/kafka/client_test.go

@@ -0,0 +1,87 @@
+package kafka
+
+import (
+	"context"
+	"os/signal"
+	"syscall"
+	"testing"
+	"time"
+
+	"ultone/internal/tool"
+
+	"github.com/loveuer/nf/nft/log"
+)
+
+func TestKafka(t *testing.T) {
+	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	defer cancel()
+
+	client, err := New("10.220.10.15:9092",
+		WithTopic("test_zyp"),
+		WithPlainAuth("admin", "Yhblsqt@!."),
+		WithReconnection(),
+	)
+	if err != nil {
+		t.Fatal(1, err)
+	}
+
+	ch, err := client.ReadMessage(ctx, ReadConfig{})
+	if err != nil {
+		t.Fatal(2, err)
+	}
+
+	for msg := range ch {
+		if msg.err != nil {
+			t.Logf("[Error] [TestKafka] msg.err = %v", msg.err)
+			continue
+		}
+
+		t.Logf("[Info ] [TestKafka] [time = %s] [msg.topic = %s] [msg.key = %s] [msg.value = %s]", time.Now().Format("060102T150405"), msg.Topic, string(msg.Key), string(msg.Value))
+	}
+}
+
+func TestKafkaWrite(t *testing.T) {
+	log.SetLogLevel(log.LogLevelDebug)
+	client, err := New("10.220.10.15:9092",
+		WithTopic("test_zyp"),
+		WithPlainAuth("admin", "Yhblsqt@!."),
+		WithReconnection(),
+	)
+	if err != nil {
+		t.Fatal(1, err)
+	}
+
+	if err = client.WriteMessages(tool.Timeout(5),
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+	); err != nil {
+		t.Log(2, err)
+	}
+
+	if err = client.WriteMessages(context.Background(),
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+		&Payload{
+			Key:   []byte(time.Now().Format("2006/01/02 15:04:05")),
+			Value: []byte(tool.RandomString(16)),
+		},
+	); err != nil {
+		t.Log(3, err)
+	}
+}

internal/database/kafka/consumer.go

@@ -0,0 +1,158 @@
+package kafka
+
+import (
+	"context"
+	"errors"
+	"io"
+	"time"
+
+	"ultone/internal/tool"
+
+	kfkgo "github.com/segmentio/kafka-go"
+)
+
+type Message struct {
+	kfkgo.Message
+	err error
+}
+
+type ReadConfig struct {
+	// MaxBytes: read buffer max bytes
+	/*
+	  - default 1MB
+	*/
+	MaxBytes int
+
+	// FirstOffset
+	/*
+		    - false: use last offset(-1)
+			- true:  use first offset(-2)
+			- default: false
+			- more: [about group offset](https://github.com/segmentio/kafka-go/blob/main/reader.go#L16)
+	*/
+	FirstOffset bool
+
+	Topic string
+	Group string
+
+	// Timeout: every read max duration
+	/*
+	  - default: 30 seconds (same with kafka-go default)
+	*/
+	Timeout int
+}
+
+var defaultReadConfig = ReadConfig{
+	// 1 MB
+	MaxBytes: 1e6,
+	Group:    "default",
+	Timeout:  30,
+}
+
+func (c *client) ReadMessage(ctx context.Context, configs ...ReadConfig) (<-chan *Message, error) {
+	var (
+		err   error
+		cfg   = ReadConfig{}
+		ch    = make(chan *Message, 1)
+		retry = 0
+	)
+
+	if len(configs) > 0 {
+		cfg = configs[0]
+	}
+
+	if cfg.Group == "" {
+		cfg.Group = defaultReadConfig.Group
+	}
+
+	if cfg.MaxBytes <= 0 {
+		cfg.MaxBytes = defaultReadConfig.MaxBytes
+	}
+
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = defaultReadConfig.Timeout
+	}
+
+	offset := kfkgo.LastOffset
+	if cfg.FirstOffset {
+		offset = kfkgo.FirstOffset
+	}
+
+	rc := kfkgo.ReaderConfig{
+		Brokers:     []string{c.address},
+		GroupID:     cfg.Group,
+		Topic:       c.topic,
+		Partition:   c.partition,
+		Dialer:      c.d,
+		MaxBytes:    cfg.MaxBytes,
+		StartOffset: offset,
+	}
+
+	if err = rc.Validate(); err != nil {
+		return nil, err
+	}
+
+	r := kfkgo.NewReader(rc)
+
+	go func() {
+		defer func() {
+			close(ch)
+			_ = r.Close()
+		}()
+	Loop:
+		for {
+			select {
+			case <-ctx.Done():
+				close(ch)
+				_ = r.Close()
+				return
+			default:
+				msg, err := r.ReadMessage(tool.TimeoutCtx(ctx, cfg.Timeout))
+				if err != nil {
+					if errors.Is(err, context.DeadlineExceeded) {
+						continue Loop
+					}
+
+					if errors.Is(err, context.Canceled) {
+						return
+					}
+
+					c.logger.Debug("kafka.ReadMessage: err = %s", err.Error())
+
+					if errors.Is(err, io.EOF) {
+						return
+					}
+
+					if errors.Is(err, io.ErrShortBuffer) {
+						ch <- &Message{
+							Message: msg,
+							err:     err,
+						}
+						continue Loop
+					}
+
+					if c.reconnection {
+						retry++
+						c.logger.Warn("kafka.ReadMessage: reconnection after 30 seconds, times = %d, err = %s", retry, err.Error())
+						time.Sleep(30 * time.Second)
+						continue Loop
+					}
+
+					ch <- &Message{
+						Message: msg,
+						err:     err,
+					}
+
+					return
+				}
+
+				ch <- &Message{
+					Message: msg,
+					err:     nil,
+				}
+			}
+		}
+	}()
+
+	return ch, nil
+}

internal/database/kafka/option.go

@@ -0,0 +1,42 @@
+package kafka
+
+import (
+	"ultone/internal/interfaces"
+
+	"github.com/segmentio/kafka-go/sasl/plain"
+)
+
+type OptionFn func(*client)
+
+func WithPlainAuth(username, password string) OptionFn {
+	return func(c *client) {
+		c.mechanism = plain.Mechanism{
+			Username: username,
+			Password: password,
+		}
+	}
+}
+
+func WithTopic(topic string) OptionFn {
+	return func(c *client) {
+		c.topic = topic
+	}
+}
+
+func WithPartition(partition int) OptionFn {
+	return func(c *client) {
+		c.partition = partition
+	}
+}
+
+func WithReconnection() OptionFn {
+	return func(c *client) {
+		c.reconnection = true
+	}
+}
+
+func WithLogger(logger interfaces.Logger) OptionFn {
+	return func(c *client) {
+		c.logger = logger
+	}
+}

internal/database/kafka/writer.go

@@ -0,0 +1,82 @@
+package kafka
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"time"
+
+	kfkgo "github.com/segmentio/kafka-go"
+)
+
+type Payload struct {
+	Key        []byte
+	Value      []byte
+	Headers    []kfkgo.Header
+	WriterData any
+}
+
+func (c *client) WriteMessages(ctx context.Context, payloads ...*Payload) error {
+	if len(payloads) == 0 {
+		return nil
+	}
+
+	times := 0
+Retry:
+	if c.writer == nil {
+		c.Lock()
+		c.writer = &kfkgo.Writer{
+			Addr:         kfkgo.TCP(c.address),
+			Topic:        c.topic,
+			Balancer:     &kfkgo.Hash{},
+			WriteTimeout: 0,
+			RequiredAcks: 0,
+			Async:        false,
+			Transport: &kfkgo.Transport{
+				DialTimeout: 30 * time.Second,
+				TLS:         &tls.Config{InsecureSkipVerify: true}, // todo
+				SASL:        c.mechanism,
+				Context:     c.ctx,
+			},
+			AllowAutoTopicCreation: true,
+		}
+		c.Unlock()
+	}
+
+	now := time.Now()
+
+	msgs := make([]kfkgo.Message, 0, len(payloads))
+
+	for _, item := range payloads {
+		msgs = append(msgs, kfkgo.Message{
+			Key:        item.Key,
+			Value:      item.Value,
+			Headers:    item.Headers,
+			WriterData: item.WriterData,
+			Time:       now,
+		})
+	}
+
+	context.WithoutCancel(ctx)
+	if err := c.writer.WriteMessages(ctx, msgs...); err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			goto HandleError
+		}
+
+		if c.reconnection {
+			times++
+			c.logger.Warn("kafka.WriteMessage: reconnection after 30 seconds, times = %d, err = %s", times, err.Error())
+			time.Sleep(30 * time.Second)
+			c.Lock()
+			c.writer = nil
+			c.Unlock()
+			goto Retry
+		}
+
+	HandleError:
+		c.logger.Warn("kafka.WriteMessage: err = %s", err.Error())
+		return err
+	}
+
+	return nil
+}