From 8bc2a2541d54435a811536aaf5fb73b6d46661c9 Mon Sep 17 00:00:00 2001 From: zhaoyupeng Date: Thu, 10 Oct 2024 18:00:49 +0800 Subject: [PATCH] wip: list files in bucket --- frontend/src/component/connection/list.tsx | 29 +++++++-- frontend/src/component/file/bucket.tsx | 40 ------------ frontend/src/component/file/content.tsx | 16 ++++- frontend/src/component/file/list.tsx | 73 ++++++++++++++++++++++ frontend/src/interfaces/connection.ts | 3 + frontend/src/store/file.tsx | 12 ++++ internal/api/api.go | 1 + internal/handler/bucket.go | 35 +++++++++++ internal/handler/connection.go | 15 +++++ internal/s3/list.go | 51 +++++++++++++++ internal/s3/new_test.go | 15 ----- internal/s3/{new.go => s3.go} | 22 +++++-- internal/s3/s3_test.go | 37 +++++++++++ 13 files changed, 279 insertions(+), 70 deletions(-) delete mode 100644 frontend/src/component/file/bucket.tsx create mode 100644 frontend/src/component/file/list.tsx create mode 100644 frontend/src/store/file.tsx create mode 100644 internal/handler/bucket.go delete mode 100644 internal/s3/new_test.go rename internal/s3/{new.go => s3.go} (69%) create mode 100644 internal/s3/s3_test.go diff --git a/frontend/src/component/connection/list.tsx b/frontend/src/component/connection/list.tsx index bed53f2..866500f 100644 --- a/frontend/src/component/connection/list.tsx +++ b/frontend/src/component/connection/list.tsx @@ -1,6 +1,16 @@ -import {Button, Input, makeStyles, MenuItem, MenuList, mergeClasses, tokens, Tooltip} from "@fluentui/react-components" +import { + Button, + Input, + makeStyles, + Menu, + MenuItem, + MenuList, MenuPopover, MenuProps, + mergeClasses, PositioningImperativeRef, + tokens, + Tooltip +} from "@fluentui/react-components" import {DatabaseLinkRegular, DismissRegular} from "@fluentui/react-icons"; -import {useState} from "react"; +import React, {useState} from "react"; import {Connection} from "../../interfaces/connection"; import {useToast} from "../../message"; import {Dial} from "../../api"; @@ -15,7 +25,6 @@ const useStyles = makeStyles({ }, content: { height: "100%", - minWidth: "22rem", width: "25rem", display: "flex", flexDirection: "column", @@ -38,7 +47,7 @@ const useStyles = makeStyles({ items_one: { marginLeft: "0.5rem", marginRight: "0.5rem", - "&:hover":{ + "&:hover": { color: tokens.colorNeutralForeground2BrandPressed, }, "&.active": { @@ -94,6 +103,11 @@ export function ConnectionList() { conn_update({...item, active: false}) } + async function handleRightClick(e: React.MouseEvent, item: Connection) { + e.preventDefault() + console.log('[DEBUG] right click connection =', item, 'event =', e) + console.log(`[DEBUG] click position: [${e.pageX}, ${e.pageY}]`) + } return (
@@ -115,14 +129,17 @@ export function ConnectionList() { {conn_list.filter(item => item.name.includes(conn_filter)).map(item => { return { await handleSelect(item) }} onDoubleClick={async () => { await handleConnect(item) }} - icon={} + onContextMenu={async (e) => { + await handleRightClick(e, item) + }} + icon={} key={item.id}> {item.name} span:nth-child(2)": { - maxWidth: '100% !important', - }, - }, -}) - -export function Bucket() { - - const styles = useStyles(); - const {bucket_list} = useStoreBucket() - - return
- - {bucket_list.map(((item, idx) => { - return } style={{width: '100%'}} - key={idx}>{item.name} - }))} - -
-} \ No newline at end of file diff --git a/frontend/src/component/file/content.tsx b/frontend/src/component/file/content.tsx index d95edf3..45b7399 100644 --- a/frontend/src/component/file/content.tsx +++ b/frontend/src/component/file/content.tsx @@ -1,6 +1,8 @@ import {Path} from "./path"; -import {Bucket} from "./bucket"; +import {ListComponent} from "./list"; import {makeStyles} from "@fluentui/react-components"; +import {useStoreBucket} from "../../store/bucket"; +import {useStoreFile} from "../../store/file"; const useStyles = makeStyles({ content: { @@ -13,9 +15,17 @@ const useStyles = makeStyles({ }) export function Content() { + const styles = useStyles() + const {bucket_list} = useStoreBucket() + const {bucket, file_list} = useStoreFile() + return
- - + + { + bucket.name ? + item.name)}/> : + item.name)}/> + }
} \ No newline at end of file diff --git a/frontend/src/component/file/list.tsx b/frontend/src/component/file/list.tsx new file mode 100644 index 0000000..a3d3198 --- /dev/null +++ b/frontend/src/component/file/list.tsx @@ -0,0 +1,73 @@ +import {makeStyles, MenuItem, MenuList, Text, tokens} from "@fluentui/react-components"; +import {ArchiveRegular, DocumentBulletListRegular} from "@fluentui/react-icons"; +import {VirtualizerScrollView} from "@fluentui/react-components/unstable"; +import React from "react"; + +const useStyles = makeStyles({ + container: { + marginTop: '0.5rem', + maxWidth: 'calc(100vw - 25rem - 1px)', + }, + row: { + height: '32px', + display: 'flex', + marginLeft: '0.5rem', + marginRight: '0.5rem', + }, + item: { + width: '100%', + maxWidth: '100%', + "&:hover": { + color: tokens.colorNeutralForeground2BrandPressed, + } + }, + text: { + overflow: 'hidden', + width: 'calc(100vw - 32rem)', + display: "block", + } +}) + +export interface ListComponentProps { + type: "bucket" | "file" + list: string[], +} + +export function ListComponent(props: ListComponentProps) { + + const styles = useStyles(); + + async function handleClick(item: string) { + console.log('[DEBUG] bucket click =', item); + } + + function handleRightClick(e: React.MouseEvent, string: string) { + e.preventDefault() + } + + return + + {(idx) => { + return
{ + await handleClick(props.list[idx]) + }} + onContextMenu={async (e) => { + handleRightClick(e, props.list[idx]) + }}> + : }> + + {props.list[idx]} + + +
+ }} + + +} \ No newline at end of file diff --git a/frontend/src/interfaces/connection.ts b/frontend/src/interfaces/connection.ts index 1402c39..a72ff9a 100644 --- a/frontend/src/interfaces/connection.ts +++ b/frontend/src/interfaces/connection.ts @@ -11,4 +11,7 @@ export interface Connection { export interface Bucket { name: string; created_at: number; +} +export interface S3File { + name:string; } \ No newline at end of file diff --git a/frontend/src/store/file.tsx b/frontend/src/store/file.tsx new file mode 100644 index 0000000..58e1e06 --- /dev/null +++ b/frontend/src/store/file.tsx @@ -0,0 +1,12 @@ +import {create} from 'zustand' +import {Bucket, S3File} from "../interfaces/connection"; + +interface StoreFile { + bucket: Bucket; + file_list: S3File[]; +} + +export const useStoreFile = create()((set) => ({ + bucket: {name: '', created_at: 0}, + file_list: [], +})) diff --git a/internal/api/api.go b/internal/api/api.go index ac4b396..9b42b4c 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -30,6 +30,7 @@ func Init(ctx context.Context) error { register("/api/connection/connect", handler.ConnectionConnect) register("/api/connection/disconnect", handler.ConnectionDisconnect) register("/api/connection/buckets", handler.ConnectionBuckets) + register("/api/bucket/file", handler.BucketFile) return nil } diff --git a/internal/handler/bucket.go b/internal/handler/bucket.go new file mode 100644 index 0000000..3f55a1b --- /dev/null +++ b/internal/handler/bucket.go @@ -0,0 +1,35 @@ +package handler + +import ( + "github.com/loveuer/nf-disk/internal/manager" + "github.com/loveuer/nf-disk/internal/s3" + "github.com/loveuer/nf-disk/ndh" +) + +func BucketFile(c *ndh.Ctx) error { + type Req struct { + ConnId uint64 `json:"conn_id"` + Bucket string `json:"bucket"` + Keyword string `json:"keyword"` + } + + var ( + err error + req = new(Req) + client *s3.Client + ) + + if err = c.ReqParse(req); err != nil { + return c.Send400(err.Error()) + } + + if req.ConnId == 0 || req.Bucket == "" { + return c.Send400(req, "缺少参数") + } + + if _, client, err = manager.Manager.Use(req.ConnId); err != nil { + return c.Send500(err.Error()) + } + + client.ListFile() +} diff --git a/internal/handler/connection.go b/internal/handler/connection.go index d85403a..c2f38d6 100644 --- a/internal/handler/connection.go +++ b/internal/handler/connection.go @@ -2,12 +2,14 @@ package handler import ( "errors" + "fmt" "github.com/loveuer/nf-disk/internal/db" "github.com/loveuer/nf-disk/internal/manager" "github.com/loveuer/nf-disk/internal/model" "github.com/loveuer/nf-disk/internal/s3" "github.com/loveuer/nf-disk/ndh" "github.com/samber/lo" + "time" ) func ConnectionTest(c *ndh.Ctx) error { @@ -209,5 +211,18 @@ func ConnectionBuckets(c *ndh.Ctx) error { return c.Send500(err.Error()) } + buckets = append(buckets, &s3.ListBucketRes{ + Name: "这是一个非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长非常长的名字", + CreatedAt: time.Now().UnixMilli(), + }) + + // todo: for frontend test + for i := 1; i <= 500; i++ { + buckets = append(buckets, &s3.ListBucketRes{ + CreatedAt: time.Now().UnixMilli(), + Name: fmt.Sprintf("test-bucket-%03d", i), + }) + } + return c.Send200(map[string]any{"list": buckets}) } diff --git a/internal/s3/list.go b/internal/s3/list.go index 0c11202..55bddbd 100644 --- a/internal/s3/list.go +++ b/internal/s3/list.go @@ -6,6 +6,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/samber/lo" + "strings" + "time" ) type ListBucketRes struct { @@ -13,6 +15,12 @@ type ListBucketRes struct { Name string `json:"name"` } +type ListFileRes struct { + Name string + LastModified time.Time + Size int64 +} + func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) { var ( err error @@ -35,3 +43,46 @@ func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) { return res, nil } + +func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, parent string) ([]*ListFileRes, error) { + var ( + err error + input = &s3.ListObjectsV2Input{ + Delimiter: aws.String("/"), + MaxKeys: aws.Int32(1000), + Bucket: aws.String(bucket), + } + output *s3.ListObjectsV2Output + ) + + if prefix != "" { + input.Prefix = aws.String(prefix) + } + + if output, err = c.client.ListObjectsV2(ctx, input); err != nil { + return nil, err + } + + folder := lo.FilterMap( + output.CommonPrefixes, + func(item types.CommonPrefix, index int) (*ListFileRes, bool) { + name := strings.TrimPrefix(*item.Prefix, parent) + return &ListFileRes{ + Name: name, + }, name != "" + }, + ) + + list := lo.Map( + output.Contents, + func(item types.Object, index int) *ListFileRes { + return &ListFileRes{ + Name: *item.Key, + LastModified: *item.LastModified, + Size: *item.Size, + } + }, + ) + + return append(folder, list...), nil +} diff --git a/internal/s3/new_test.go b/internal/s3/new_test.go deleted file mode 100644 index 52ce52a..0000000 --- a/internal/s3/new_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package s3 - -import ( - "context" - "github.com/loveuer/nf/nft/log" - "testing" -) - -func TestNewClient(t *testing.T) { - log.SetLogLevel(log.LogLevelDebug) - _, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H") - if err != nil { - t.Fatalf("call s3.New err = %s", err.Error()) - } -} diff --git a/internal/s3/new.go b/internal/s3/s3.go similarity index 69% rename from internal/s3/new.go rename to internal/s3/s3.go index ae8dd38..4772662 100644 --- a/internal/s3/new.go +++ b/internal/s3/s3.go @@ -14,11 +14,10 @@ import ( type resolverV2 struct{} -func (*resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) ( - smithyendpoints.Endpoint, error, -) { +func (*resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { u, err := url.Parse(*params.Endpoint) if err != nil { + log.Warn("resolver v2: parse url = %s, err = %s", params.Endpoint, err.Error()) return smithyendpoints.Endpoint{}, err } return smithyendpoints.Endpoint{ @@ -38,14 +37,25 @@ func New(ctx context.Context, endpoint string, access string, key string) (*Clie output *s3.ListBucketsOutput ) - if sdkConfig, err = config.LoadDefaultConfig(ctx); err != nil { + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + }, nil + }) + + if sdkConfig, err = config.LoadDefaultConfig( + ctx, + config.WithEndpointResolverWithOptions(customResolver), + ); err != nil { return nil, err } s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) { - o.BaseEndpoint = aws.String(endpoint) - o.EndpointResolverV2 = &resolverV2{} + //o.BaseEndpoint = aws.String(endpoint) + //o.EndpointResolverV2 = &resolverV2{} o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(access, key, "")) + o.UsePathStyle = true + o.Region = "auto" }) if output, err = s3Client.ListBuckets(tool.Timeout(5), &s3.ListBucketsInput{ diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go new file mode 100644 index 0000000..8b6aabe --- /dev/null +++ b/internal/s3/s3_test.go @@ -0,0 +1,37 @@ +package s3 + +import ( + "context" + "github.com/loveuer/nf-disk/internal/tool" + "github.com/loveuer/nf/nft/log" + "testing" +) + +func TestNewClient(t *testing.T) { + log.SetLogLevel(log.LogLevelDebug) + _, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H") + if err != nil { + t.Fatalf("call s3.New err = %s", err.Error()) + } +} + +func TestListFile(t *testing.T) { + //log.SetLogLevel(log.LogLevelDebug) + + //cli, err := New(context.TODO(), "http://10.220.10.14:19000", "5VCR05L4BSGNCTCD8DXP", "FPTMYBEiHhWLJ05C3aGXW8bjFXXNmghc8Za3Fo2u") + cli, err := New(context.TODO(), "http://10.220.10.15:9000/", "8ALV3DUZI31YG4BDRJ0Z", "CRqwS1MsiUj27TbRK+3T2n+LpKWd07VvaDKuzU0H") + if err != nil { + t.Fatalf("call s3.New err = %s", err.Error()) + } + + files, err := cli.ListFile(tool.Timeout(30), "infobox-person", "") + if err != nil { + t.Fatalf("call s3.ListFile err = %s", err.Error()) + } + + t.Logf("[x] file length = %d", len(files)) + + for _, item := range files { + t.Logf("[x] file = %s, size = %d", item.Name, item.Size) + } +}