loveuer/nf-disk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: list files in bucket

Browse Source
This commit is contained in:
zhaoyupeng
2024-10-10 18:00:49 +08:00
parent efe7800b59
commit 8bc2a2541d
13 changed files with 279 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/api/api.go
View File

@ -30,6 +30,7 @@ func Init(ctx context.Context) error {
register("/api/connection/connect", handler.ConnectionConnect)
register("/api/connection/disconnect", handler.ConnectionDisconnect)
register("/api/connection/buckets", handler.ConnectionBuckets)
register("/api/bucket/file", handler.BucketFile)
return nil
}

35
internal/handler/bucket.go Normal file
View File

@ -0,0 +1,35 @@
package handler
import (
"github.com/loveuer/nf-disk/internal/manager"
"github.com/loveuer/nf-disk/internal/s3"
"github.com/loveuer/nf-disk/ndh"
)
func BucketFile(c *ndh.Ctx) error {
type Req struct {
ConnId uint64 `json:"conn_id"`
Bucket string `json:"bucket"`
Keyword string `json:"keyword"`
}
var (
err error
req = new(Req)
client *s3.Client
)
if err = c.ReqParse(req); err != nil {
return c.Send400(err.Error())
}
if req.ConnId == 0 || req.Bucket == "" {
return c.Send400(req, "缺少参数")
}
if _, client, err = manager.Manager.Use(req.ConnId); err != nil {
return c.Send500(err.Error())
}
client.ListFile()
}

15
internal/handler/connection.go
View File

@ -2,12 +2,14 @@ package handler
import (
"errors"
"fmt"
"github.com/loveuer/nf-disk/internal/db"
"github.com/loveuer/nf-disk/internal/manager"
"github.com/loveuer/nf-disk/internal/model"
"github.com/loveuer/nf-disk/internal/s3"
"github.com/loveuer/nf-disk/ndh"
"github.com/samber/lo"
"time"
)
func ConnectionTest(c *ndh.Ctx) error {
@ -209,5 +211,18 @@ func ConnectionBuckets(c *ndh.Ctx) error {
return c.Send500(err.Error())
}
buckets = append(buckets, &s3.ListBucketRes{
Name: "这是一个非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长的名字",
CreatedAt: time.Now().UnixMilli(),
})
// todo: for frontend test
for i := 1; i <= 500; i++ {
buckets = append(buckets, &s3.ListBucketRes{
CreatedAt: time.Now().UnixMilli(),
Name: fmt.Sprintf("test-bucket-%03d", i),
})
}
return c.Send200(map[string]any{"list": buckets})
}

51
internal/s3/list.go
View File

@ -6,6 +6,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/samber/lo"
"strings"
"time"
)
type ListBucketRes struct {
@ -13,6 +15,12 @@ type ListBucketRes struct {
Name string `json:"name"`
}
type ListFileRes struct {
Name string
LastModified time.Time
Size int64
}
func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) {
var (
err error
@ -35,3 +43,46 @@ func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) {
return res, nil
}
func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, parent string) ([]*ListFileRes, error) {
var (
err error
input = &s3.ListObjectsV2Input{
Delimiter: aws.String("/"),
MaxKeys: aws.Int32(1000),
Bucket: aws.String(bucket),
}
output *s3.ListObjectsV2Output
)
if prefix != "" {
input.Prefix = aws.String(prefix)
}
if output, err = c.client.ListObjectsV2(ctx, input); err != nil {
return nil, err
}
folder := lo.FilterMap(
output.CommonPrefixes,
func(item types.CommonPrefix, index int) (*ListFileRes, bool) {
name := strings.TrimPrefix(*item.Prefix, parent)
return &ListFileRes{
Name: name,
}, name != ""
},
)
list := lo.Map(
output.Contents,
func(item types.Object, index int) *ListFileRes {
return &ListFileRes{
Name: *item.Key,
LastModified: *item.LastModified,
Size: *item.Size,
}
},
)
return append(folder, list...), nil
}

15
internal/s3/new_test.go
View File

@ -1,15 +0,0 @@
package s3
import (
"context"
"github.com/loveuer/nf/nft/log"
"testing"
)
func TestNewClient(t *testing.T) {
log.SetLogLevel(log.LogLevelDebug)
_, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H")
if err != nil {
t.Fatalf("call s3.New err = %s", err.Error())
}
}

22
internal/s3/new.go → internal/s3/s3.go
View File

@ -14,11 +14,10 @@ import (
type resolverV2 struct{}
func (*resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (
smithyendpoints.Endpoint, error,
) {
func (*resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) {
u, err := url.Parse(*params.Endpoint)
if err != nil {
log.Warn("resolver v2: parse url = %s, err = %s", params.Endpoint, err.Error())
return smithyendpoints.Endpoint{}, err
}
return smithyendpoints.Endpoint{
@ -38,14 +37,25 @@ func New(ctx context.Context, endpoint string, access string, key string) (*Clie
output *s3.ListBucketsOutput
)
if sdkConfig, err = config.LoadDefaultConfig(ctx); err != nil {
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: endpoint,
}, nil
})
if sdkConfig, err = config.LoadDefaultConfig(
ctx,
config.WithEndpointResolverWithOptions(customResolver),
); err != nil {
return nil, err
}
s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.EndpointResolverV2 = &resolverV2{}
//o.BaseEndpoint = aws.String(endpoint)
//o.EndpointResolverV2 = &resolverV2{}
o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(access, key, ""))
o.UsePathStyle = true
o.Region = "auto"
})
if output, err = s3Client.ListBuckets(tool.Timeout(5), &s3.ListBucketsInput{

37
internal/s3/s3_test.go Normal file
View File

@ -0,0 +1,37 @@
package s3
import (
"context"
"github.com/loveuer/nf-disk/internal/tool"
"github.com/loveuer/nf/nft/log"
"testing"
)
func TestNewClient(t *testing.T) {
log.SetLogLevel(log.LogLevelDebug)
_, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H")
if err != nil {
t.Fatalf("call s3.New err = %s", err.Error())
}
}
func TestListFile(t *testing.T) {
//log.SetLogLevel(log.LogLevelDebug)
//cli, err := New(context.TODO(), "http://10.220.10.14:19000", "5VCR05L4BSGNCTCD8DXP", "FPTMYBEiHhWLJ05C3aGXW8bjFXXNmghc8Za3Fo2u")
cli, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H")
if err != nil {
t.Fatalf("call s3.New err = %s", err.Error())
}
files, err := cli.ListFile(tool.Timeout(30), "infobox-person", "")
if err != nil {
t.Fatalf("call s3.ListFile err = %s", err.Error())
}
t.Logf("[x] file length = %d", len(files))
for _, item := range files {
t.Logf("[x] file = %s, size = %d", item.Name, item.Size)
}
}