🎉: init project

This commit is contained in:
loveuer
2024-07-11 16:37:26 +08:00
commit c46458c6f2
159 changed files with 19246 additions and 0 deletions

109
internal/database/cache/cache_lru.go vendored Normal file
View File

@ -0,0 +1,109 @@
package cache
import (
"context"
"github.com/hashicorp/golang-lru/v2/expirable"
_ "github.com/hashicorp/golang-lru/v2/expirable"
"time"
"ultone/internal/interfaces"
)
var _ interfaces.Cacher = (*_lru)(nil)
type _lru struct {
client *expirable.LRU[string, *_lru_value]
}
type _lru_value struct {
duration time.Duration
last time.Time
bs []byte
}
func (l *_lru) Get(ctx context.Context, key string) ([]byte, error) {
v, ok := l.client.Get(key)
if !ok {
return nil, ErrorKeyNotFound
}
if v.duration == 0 {
return v.bs, nil
}
if time.Now().Sub(v.last) > v.duration {
l.client.Remove(key)
return nil, ErrorKeyNotFound
}
return v.bs, nil
}
func (l *_lru) GetEx(ctx context.Context, key string, duration time.Duration) ([]byte, error) {
v, ok := l.client.Get(key)
if !ok {
return nil, ErrorKeyNotFound
}
if v.duration == 0 {
return v.bs, nil
}
now := time.Now()
if now.Sub(v.last) > v.duration {
l.client.Remove(key)
return nil, ErrorKeyNotFound
}
l.client.Add(key, &_lru_value{
duration: duration,
last: now,
bs: v.bs,
})
return v.bs, nil
}
func (l *_lru) Set(ctx context.Context, key string, value any) error {
bs, err := handleValue(value)
if err != nil {
return err
}
l.client.Add(key, &_lru_value{
duration: 0,
last: time.Now(),
bs: bs,
})
return nil
}
func (l *_lru) SetEx(ctx context.Context, key string, value any, duration time.Duration) error {
bs, err := handleValue(value)
if err != nil {
return err
}
l.client.Add(key, &_lru_value{
duration: duration,
last: time.Now(),
bs: bs,
})
return nil
}
func (l *_lru) Del(ctx context.Context, keys ...string) error {
for _, key := range keys {
l.client.Remove(key)
}
return nil
}
func newLRUCache() (interfaces.Cacher, error) {
client := expirable.NewLRU[string, *_lru_value](0, nil, 0)
return &_lru{client: client}, nil
}

65
internal/database/cache/cache_memory.go vendored Normal file
View File

@ -0,0 +1,65 @@
package cache
import (
"context"
"fmt"
"time"
"ultone/internal/interfaces"
"gitea.com/taozitaozi/gredis"
)
var _ interfaces.Cacher = (*_mem)(nil)
type _mem struct {
client *gredis.Gredis
}
func (m *_mem) Get(ctx context.Context, key string) ([]byte, error) {
v, err := m.client.Get(key)
if err != nil {
return nil, err
}
bs, ok := v.([]byte)
if !ok {
return nil, fmt.Errorf("invalid value type=%T", v)
}
return bs, nil
}
func (m *_mem) GetEx(ctx context.Context, key string, duration time.Duration) ([]byte, error) {
v, err := m.client.GetEx(key, duration)
if err != nil {
return nil, err
}
bs, ok := v.([]byte)
if !ok {
return nil, fmt.Errorf("invalid value type=%T", v)
}
return bs, nil
}
func (m *_mem) Set(ctx context.Context, key string, value any) error {
bs, err := handleValue(value)
if err != nil {
return err
}
return m.client.Set(key, bs)
}
func (m *_mem) SetEx(ctx context.Context, key string, value any, duration time.Duration) error {
bs, err := handleValue(value)
if err != nil {
return err
}
return m.client.SetEx(key, bs, duration)
}
func (m *_mem) Del(ctx context.Context, keys ...string) error {
m.client.Delete(keys...)
return nil
}

54
internal/database/cache/cache_redis.go vendored Normal file
View File

@ -0,0 +1,54 @@
package cache
import (
"context"
"github.com/go-redis/redis/v8"
"time"
)
type _redis struct {
client *redis.Client
}
func (r *_redis) Get(ctx context.Context, key string) ([]byte, error) {
result, err := r.client.Get(ctx, key).Result()
if err != nil {
return nil, err
}
return []byte(result), nil
}
func (r *_redis) GetEx(ctx context.Context, key string, duration time.Duration) ([]byte, error) {
result, err := r.client.GetEx(ctx, key, duration).Result()
if err != nil {
return nil, err
}
return []byte(result), nil
}
func (r *_redis) Set(ctx context.Context, key string, value any) error {
bs, err := handleValue(value)
if err != nil {
return err
}
_, err = r.client.Set(ctx, key, bs, redis.KeepTTL).Result()
return err
}
func (r *_redis) SetEx(ctx context.Context, key string, value any, duration time.Duration) error {
bs, err := handleValue(value)
if err != nil {
return err
}
_, err = r.client.SetEX(ctx, key, bs, duration).Result()
return err
}
func (r *_redis) Del(ctx context.Context, keys ...string) error {
return r.client.Del(ctx, keys...).Err()
}

33
internal/database/cache/client.go vendored Normal file
View File

@ -0,0 +1,33 @@
package cache
import (
"encoding/json"
"ultone/internal/interfaces"
)
var (
Client interfaces.Cacher
)
type encoded_value interface {
MarshalBinary() ([]byte, error)
}
type decoded_value interface {
UnmarshalBinary(bs []byte) error
}
func handleValue(value any) ([]byte, error) {
var (
bs []byte
err error
)
if imp, ok := value.(encoded_value); ok {
bs, err = imp.MarshalBinary()
} else {
bs, err = json.Marshal(value)
}
return bs, err
}

7
internal/database/cache/error.go vendored Normal file
View File

@ -0,0 +1,7 @@
package cache
import "errors"
var (
ErrorKeyNotFound = errors.New("key not found")
)

70
internal/database/cache/init.go vendored Normal file
View File

@ -0,0 +1,70 @@
package cache
import (
"fmt"
"gitea.com/taozitaozi/gredis"
"github.com/go-redis/redis/v8"
"net/url"
"strings"
"ultone/internal/opt"
"ultone/internal/tool"
)
func Init() error {
var (
err error
)
strs := strings.Split(opt.Cfg.Cache.Uri, "::")
switch strs[0] {
case "memory":
gc := gredis.NewGredis(1024 * 1024)
Client = &_mem{client: gc}
case "lru":
if Client, err = newLRUCache(); err != nil {
return err
}
case "redis":
var (
ins *url.URL
err error
)
if len(strs) != 2 {
return fmt.Errorf("cache.Init: invalid cache uri: %s", opt.Cfg.Cache.Uri)
}
uri := strs[1]
if !strings.Contains(uri, "://") {
uri = fmt.Sprintf("redis://%s", uri)
}
if ins, err = url.Parse(uri); err != nil {
return fmt.Errorf("cache.Init: url parse cache uri: %s, err: %s", opt.Cfg.Cache.Uri, err.Error())
}
addr := ins.Host
username := ins.User.Username()
password, _ := ins.User.Password()
var rc *redis.Client
rc = redis.NewClient(&redis.Options{
Addr: addr,
Username: username,
Password: password,
})
if err = rc.Ping(tool.Timeout(5)).Err(); err != nil {
return fmt.Errorf("cache.Init: redis ping err: %s", err.Error())
}
Client = &_redis{client: rc}
default:
return fmt.Errorf("cache type %s not support", strs[0])
}
return nil
}

View File

@ -0,0 +1,38 @@
package db
import (
"context"
"gorm.io/gorm"
"ultone/internal/opt"
"ultone/internal/tool"
)
var (
cli = &client{}
)
type client struct {
cli *gorm.DB
ttype string
}
func Type() string {
return cli.ttype
}
func New(ctxs ...context.Context) *gorm.DB {
var ctx context.Context
if len(ctxs) > 0 && ctxs[0] != nil {
ctx = ctxs[0]
} else {
ctx = tool.Timeout(30)
}
session := cli.cli.Session(&gorm.Session{Context: ctx})
if opt.Debug {
session = session.Debug()
}
return session
}

View File

@ -0,0 +1,46 @@
package db
import (
"fmt"
"github.com/glebarez/sqlite"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"strings"
"ultone/internal/opt"
)
func Init() error {
strs := strings.Split(opt.Cfg.DB.Uri, "::")
if len(strs) != 2 {
return fmt.Errorf("db.Init: opt db uri invalid: %s", opt.Cfg.DB.Uri)
}
cli.ttype = strs[0]
var (
err error
dsn = strs[1]
)
switch strs[0] {
case "sqlite":
opt.Cfg.DB.Type = "sqlite"
cli.cli, err = gorm.Open(sqlite.Open(dsn))
case "mysql":
opt.Cfg.DB.Type = "mysql"
cli.cli, err = gorm.Open(mysql.Open(dsn))
case "postgres":
opt.Cfg.DB.Type = "postgres"
cli.cli, err = gorm.Open(postgres.Open(dsn))
default:
return fmt.Errorf("db type only support: [sqlite, mysql, postgres], unsupported db type: %s", strs[0])
}
if err != nil {
return fmt.Errorf("db.Init: open %s with dsn:%s, err: %w", strs[0], dsn, err)
}
return nil
}

View File

@ -0,0 +1,29 @@
package es
import (
elastic "github.com/elastic/go-elasticsearch/v7"
"github.com/loveuer/esgo2dump/xes/es7"
"github.com/loveuer/nf/nft/log"
"net/url"
"ultone/internal/opt"
"ultone/internal/tool"
)
var (
Client *elastic.Client
)
func Init() error {
ins, err := url.Parse(opt.Cfg.ES.Uri)
if err != nil {
return err
}
log.Debug("es.InitClient url parse uri: %s, result: %+v", opt.Cfg.ES.Uri, ins)
if Client, err = es7.NewClient(tool.Timeout(10), ins); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,89 @@
package mq
import (
"crypto/tls"
"fmt"
amqp "github.com/rabbitmq/amqp091-go"
"net/url"
"sync"
)
// Init - init mq client:
// - @param.uri: "{scheme[amqp/amqps]}://{username}:{password}@{endpoint}/{virtual_host}"
// - @param.certs: with amqps, certs[0]=client crt bytes, certs[0]=client key bytes
type _client struct {
sync.Mutex
uri string
tlsCfg *tls.Config
conn *amqp.Connection
ch *amqp.Channel
consume <-chan amqp.Delivery
queue *queueOption
}
func (c *_client) open() error {
var (
err error
)
c.Lock()
defer c.Unlock()
if c.tlsCfg != nil {
c.conn, err = amqp.DialTLS(c.uri, c.tlsCfg)
} else {
c.conn, err = amqp.Dial(c.uri)
}
if err != nil {
return err
}
if c.ch, err = c.conn.Channel(); err != nil {
return err
}
if client.queue != nil && client.queue.name != "" {
if _, err = client.ch.QueueDeclare(
client.queue.name,
client.queue.durable,
client.queue.autoDelete,
client.queue.exclusive,
client.queue.noWait,
client.queue.args,
); err != nil {
return fmt.Errorf("declare queue: %s, err: %w", client.queue.name, err)
}
}
return nil
}
var (
client = &_client{
uri: "amqp://guest:guest@127.0.0.1:5672/",
tlsCfg: nil,
}
)
// Init - init mq client
func Init(opts ...OptionFn) error {
var (
err error
)
for _, fn := range opts {
fn(client)
}
if _, err = url.Parse(client.uri); err != nil {
return fmt.Errorf("url parse uri err: %w", err)
}
if err = client.open(); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,111 @@
package mq
import (
"context"
"crypto/tls"
"crypto/x509"
amqp "github.com/rabbitmq/amqp091-go"
"os"
"os/signal"
"strconv"
"syscall"
"testing"
"time"
)
func TestConsume(t *testing.T) {
clientCert, err := tls.LoadX509KeyPair(
"/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/client.crt",
"/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/client.key",
)
if err != nil {
t.Fatal(err.Error())
}
ca, err := os.ReadFile("/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/ca.crt")
if err != nil {
t.Fatal(err.Error())
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(ca) {
t.Fatal("ca pool append ca crt err")
}
if err := Init(
WithURI("amqps://admin:password@mq.dev:5671/export"),
WithTLS(&tls.Config{
Certificates: []tls.Certificate{clientCert},
RootCAs: caCertPool,
InsecureSkipVerify: true,
}),
WithQueueDeclare("export", false, false, false, false, amqp.Table{"x-max-priority": 100}),
); err != nil {
t.Fatal(err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
defer cancel()
ch, err := Consume(ctx, "export", &ConsumeOpt{MaxReconnection: -1})
if err != nil {
t.Fatal(err)
}
t.Log("[TEST] start consume msg")
for msg := range ch {
t.Logf("[TEST] [%s] [msg: %s]", time.Now().Format("060102T150405"), string(msg.Body))
_ = msg.Ack(false)
}
}
func TestPublish(t *testing.T) {
clientCert, err := tls.LoadX509KeyPair(
"/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/client.crt",
"/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/client.key",
)
if err != nil {
t.Fatal(err.Error())
}
ca, err := os.ReadFile("/Users/loveuer/codes/project/bifrost-pro/search_v3/internal/database/mq/tls/ca.crt")
if err != nil {
t.Fatal(err.Error())
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(ca) {
t.Fatal("ca pool append ca crt err")
}
if err := Init(
WithURI("amqps://admin:password@mq.dev:5671/export"),
WithTLS(&tls.Config{
Certificates: []tls.Certificate{clientCert},
RootCAs: caCertPool,
InsecureSkipVerify: true,
}),
WithQueueDeclare("export", false, false, false, false, amqp.Table{"x-max-priority": 100}),
); err != nil {
t.Fatal(err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
defer cancel()
count := 1
t.Log("[TEST] start publish msg...")
for {
if err = Publish(ctx, "export", amqp.Publishing{
ContentType: "text/plain",
Body: []byte(time.Now().Format(time.RFC3339) + " => hello_world@" + strconv.Itoa(count)),
}); err != nil {
t.Log(err.Error())
}
time.Sleep(11 * time.Second)
count++
}
}

View File

@ -0,0 +1,97 @@
package mq
import (
"context"
"fmt"
"github.com/loveuer/esgo2dump/log"
amqp "github.com/rabbitmq/amqp091-go"
"os"
"time"
"ultone/internal/tool"
)
// ConsumeOpt
// - Name: consumer's name, default unamed_<timestamp>
// - MaxReconnection: when mq connection closed, max reconnection times, default 3, -1 for unlimited
type ConsumeOpt struct {
Name string // consumer's name, default unamed_<timestamp>
AutoAck bool
Exclusive bool
NoLocal bool
NoWait bool
MaxReconnection int // when mq connection closed, max reconnection times, default 3, -1 for unlimited
Args amqp.Table
}
func Consume(ctx context.Context, queue string, opts ...*ConsumeOpt) (<-chan amqp.Delivery, error) {
var (
err error
res = make(chan amqp.Delivery, 1)
opt = &ConsumeOpt{
Name: os.Getenv("HOSTNAME"),
AutoAck: false,
Exclusive: false,
NoLocal: false,
NoWait: false,
Args: nil,
MaxReconnection: 3,
}
)
if len(opts) > 0 && opts[0] != nil {
opt = opts[0]
}
if opt.Name == "" {
opt.Name = fmt.Sprintf("unamed_%d", time.Now().UnixMilli())
}
client.Lock()
if client.consume, err = client.ch.Consume(queue, opt.Name, opt.AutoAck, opt.Exclusive, opt.NoLocal, opt.NoWait, opt.Args); err != nil {
client.Unlock()
return nil, err
}
client.Unlock()
go func() {
Run:
retry := 0
for {
select {
case <-ctx.Done():
close(res)
return
case m, ok := <-client.consume:
if !ok {
log.Warn("[mq] consume channel closed!!!")
goto Reconnect
}
res <- m
}
}
Reconnect:
if opt.MaxReconnection == -1 || opt.MaxReconnection > retry {
retry++
log.Warn("[mq] try reconnect[%d/%d] to mq server after %d seconds...err: %v", retry, opt.MaxReconnection, tool.Min(60, retry*5), err)
time.Sleep(time.Duration(tool.Min(60, retry*5)) * time.Second)
if err = client.open(); err != nil {
goto Reconnect
}
client.Lock()
if client.consume, err = client.ch.Consume(queue, opt.Name, opt.AutoAck, opt.Exclusive, opt.NoLocal, opt.NoWait, opt.Args); err != nil {
client.Unlock()
goto Reconnect
}
client.Unlock()
log.Info("[mq] reconnect success!!!")
goto Run
}
}()
return res, nil
}

View File

@ -0,0 +1,48 @@
package mq
import (
"crypto/tls"
amqp "github.com/rabbitmq/amqp091-go"
)
type OptionFn func(*_client)
// WithURI
// - amqp uri, e.g. amqp://guest:guest@127.0.0.1:5672/vhost
// - tips: with tls, scheme should be amqps, amqps://xx:xx@127.0.0.1:5672/vhost
func WithURI(uri string) OptionFn {
return func(c *_client) {
c.uri = uri
}
}
// WithTLS
// - amqps tls config
// - include client cert, client key, ca cert
func WithTLS(tlsCfg *tls.Config) OptionFn {
return func(c *_client) {
c.tlsCfg = tlsCfg
}
}
type queueOption struct {
name string
durable bool
autoDelete bool
exclusive bool
noWait bool
args amqp.Table
}
func WithQueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) OptionFn {
return func(c *_client) {
c.queue = &queueOption{
name: name,
durable: durable,
autoDelete: autoDelete,
exclusive: exclusive,
noWait: noWait,
args: args,
}
}
}

View File

@ -0,0 +1,62 @@
package mq
import (
"context"
"errors"
"github.com/loveuer/esgo2dump/log"
amqp "github.com/rabbitmq/amqp091-go"
"time"
"ultone/internal/tool"
)
// PublishOpt
// - MaxReconnect: publish msg auto retry with reconnect, should not be big, case memory leak
type PublishOpt struct {
Exchange string
Mandatory bool
Immediate bool
MaxReconnect uint8 // publish msg auto retry with reconnect, should not be big(default 1), case memory leak
}
func Publish(ctx context.Context, queue string, msg amqp.Publishing, opts ...*PublishOpt) error {
var (
err error
opt = &PublishOpt{
Exchange: amqp.DefaultExchange,
Mandatory: false,
Immediate: false,
MaxReconnect: 1,
}
retry = 0
)
if len(opts) > 0 && opts[0] != nil {
opt = opts[0]
}
for ; retry <= int(opt.MaxReconnect); retry++ {
if err = client.ch.PublishWithContext(ctx, opt.Exchange, queue, opt.Mandatory, opt.Immediate, msg); err == nil {
return nil
}
if errors.Is(err, amqp.ErrClosed) {
sleep := tool.Min(120, (retry+1)*30)
log.Warn("[mq] connection closed, reconnect[%d/%d] after %d seconds", retry+1, opt.MaxReconnect, sleep)
time.Sleep(time.Duration(sleep) * time.Second)
if oerr := client.open(); oerr != nil {
log.Error("[mq] reconnect[%d/%d] mq err: %v", oerr, retry+1, opt.MaxReconnect)
} else {
log.Info("[mq] reconnect mq success!!!")
}
continue
}
return err
}
return err
}

View File

@ -0,0 +1,34 @@
package nebula
import (
"context"
"github.com/loveuer/ngorm/v2"
"strings"
"ultone/internal/opt"
)
var (
client *ngorm.Client
)
func Init(ctx context.Context, cfg opt.Nebula) error {
var (
err error
)
if client, err = ngorm.NewClient(ctx, &ngorm.Config{
Endpoints: strings.Split(cfg.Uri, ","),
Username: cfg.Username,
Password: cfg.Password,
DefaultSpace: cfg.Space,
Logger: nil,
}); err != nil {
return err
}
return nil
}
func New(ctx context.Context, cfgs ...*ngorm.SessCfg) *ngorm.Session {
return client.Session(cfgs...)
}