From f54ed67f0f9783c4856d07ce2c64931b6413b461 Mon Sep 17 00:00:00 2001 From: zhaoyupeng Date: Fri, 11 Oct 2024 18:03:09 +0800 Subject: [PATCH] wip: s3 file prefix filter --- frontend/package-lock.json | 19 ++++ frontend/package.json | 1 + frontend/src/component/connection/list.tsx | 13 ++- frontend/src/component/file/content.tsx | 13 +-- .../file/{list.tsx => list_bucket.tsx} | 31 ++++--- frontend/src/component/file/list_file.tsx | 79 +++++++++++++++++ frontend/src/component/file/path.tsx | 86 ++++++++++++++++++- frontend/src/interfaces/connection.ts | 7 +- frontend/src/store/bucket.tsx | 18 ++-- frontend/src/store/file.tsx | 27 ++++-- internal/api/api.go | 2 +- internal/handler/bucket.go | 13 ++- internal/s3/list.go | 25 ++++-- internal/s3/s3_test.go | 4 +- 14 files changed, 286 insertions(+), 52 deletions(-) rename frontend/src/component/file/{list.tsx => list_bucket.tsx} (67%) create mode 100644 frontend/src/component/file/list_file.tsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7a98e39..ccf5337 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -15,6 +15,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.26.2", + "use-debounce": "^10.0.3", "zustand": "^5.0.0-rc.2" }, "devDependencies": { @@ -3465,6 +3466,18 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/use-debounce": { + "version": "10.0.3", + "resolved": "https://registry.npmmirror.com/use-debounce/-/use-debounce-10.0.3.tgz", + "integrity": "sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==", + "license": "MIT", + "engines": { + "node": ">= 16.0.0" + }, + "peerDependencies": { + "react": "*" + } + }, "node_modules/use-disposable": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/use-disposable/-/use-disposable-1.0.2.tgz", @@ -5915,6 +5928,12 @@ "picocolors": "^1.0.1" } }, + "use-debounce": { + "version": "10.0.3", + "resolved": "https://registry.npmmirror.com/use-debounce/-/use-debounce-10.0.3.tgz", + "integrity": "sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==", + "requires": {} + }, "use-disposable": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/use-disposable/-/use-disposable-1.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 5321ffd..33b50c4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,6 +16,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.26.2", + "use-debounce": "^10.0.3", "zustand": "^5.0.0-rc.2" }, "devDependencies": { diff --git a/frontend/src/component/connection/list.tsx b/frontend/src/component/connection/list.tsx index 866500f..3a3aaa9 100644 --- a/frontend/src/component/connection/list.tsx +++ b/frontend/src/component/connection/list.tsx @@ -11,7 +11,7 @@ import { } from "@fluentui/react-components" import {DatabaseLinkRegular, DismissRegular} from "@fluentui/react-icons"; import React, {useState} from "react"; -import {Connection} from "../../interfaces/connection"; +import {Bucket, Connection} from "../../interfaces/connection"; import {useToast} from "../../message"; import {Dial} from "../../api"; import {useStoreConnection} from "../../store/connection"; @@ -77,10 +77,16 @@ export function ConnectionList() { const {dispatchMessage} = useToast(); const {conn_list, conn_update} = useStoreConnection(); const [conn_filter, set_conn_filter] = useState(''); - const {bucket_get} = useStoreBucket() + const {bucket_get, bucket_set} = useStoreBucket() async function handleSelect(item: Connection) { - + conn_list.map((one: Connection) => { + if (item.id === one.id && one.active) { + conn_update(one) + bucket_get(one, false) + bucket_set(null) + } + }) } async function handleConnect(item: Connection) { @@ -92,6 +98,7 @@ export function ConnectionList() { conn_update({...item, active: true}) bucket_get(item, true) + bucket_set(null) } async function handleDisconnect(item: Connection) { diff --git a/frontend/src/component/file/content.tsx b/frontend/src/component/file/content.tsx index 45b7399..a1333c9 100644 --- a/frontend/src/component/file/content.tsx +++ b/frontend/src/component/file/content.tsx @@ -1,8 +1,9 @@ import {Path} from "./path"; -import {ListComponent} from "./list"; +import {ListBucketComponent} from "./list_bucket"; import {makeStyles} from "@fluentui/react-components"; import {useStoreBucket} from "../../store/bucket"; import {useStoreFile} from "../../store/file"; +import {ListFileComponent} from "./list_file"; const useStyles = makeStyles({ content: { @@ -17,15 +18,15 @@ const useStyles = makeStyles({ export function Content() { const styles = useStyles() - const {bucket_list} = useStoreBucket() - const {bucket, file_list} = useStoreFile() + const {bucket_active, bucket_list} = useStoreBucket() + const {file_list} = useStoreFile() return
{ - bucket.name ? - item.name)}/> : - item.name)}/> + bucket_active ? + : + }
} \ No newline at end of file diff --git a/frontend/src/component/file/list.tsx b/frontend/src/component/file/list_bucket.tsx similarity index 67% rename from frontend/src/component/file/list.tsx rename to frontend/src/component/file/list_bucket.tsx index a3d3198..a995da6 100644 --- a/frontend/src/component/file/list.tsx +++ b/frontend/src/component/file/list_bucket.tsx @@ -2,6 +2,10 @@ import {makeStyles, MenuItem, MenuList, Text, tokens} from "@fluentui/react-comp import {ArchiveRegular, DocumentBulletListRegular} from "@fluentui/react-icons"; import {VirtualizerScrollView} from "@fluentui/react-components/unstable"; import React from "react"; +import {useStoreBucket} from "../../store/bucket"; +import {Bucket} from "../../interfaces/connection"; +import {useStoreFile} from "../../store/file"; +import {useStoreConnection} from "../../store/connection"; const useStyles = makeStyles({ container: { @@ -28,26 +32,25 @@ const useStyles = makeStyles({ } }) -export interface ListComponentProps { - type: "bucket" | "file" - list: string[], -} - -export function ListComponent(props: ListComponentProps) { +export function ListBucketComponent() { const styles = useStyles(); + const {conn_active} = useStoreConnection() + const {bucket_set, bucket_list} = useStoreBucket() + const {files_get} = useStoreFile() - async function handleClick(item: string) { - console.log('[DEBUG] bucket click =', item); + async function handleClick(item: Bucket) { + bucket_set(item) + files_get(conn_active!, item, "") } - function handleRightClick(e: React.MouseEvent, string: string) { + function handleRightClick(e: React.MouseEvent, item: Bucket) { e.preventDefault() } return @@ -55,15 +58,15 @@ export function ListComponent(props: ListComponentProps) { return
{ - await handleClick(props.list[idx]) + await handleClick(bucket_list[idx]) }} onContextMenu={async (e) => { - handleRightClick(e, props.list[idx]) + handleRightClick(e, bucket_list[idx]) }}> : }> + icon={}> - {props.list[idx]} + {bucket_list[idx].name}
diff --git a/frontend/src/component/file/list_file.tsx b/frontend/src/component/file/list_file.tsx new file mode 100644 index 0000000..32e801f --- /dev/null +++ b/frontend/src/component/file/list_file.tsx @@ -0,0 +1,79 @@ +import {makeStyles, MenuItem, MenuList, Text, tokens} from "@fluentui/react-components"; +import {ArchiveRegular, DocumentBulletListRegular, FolderRegular} from "@fluentui/react-icons"; +import {VirtualizerScrollView} from "@fluentui/react-components/unstable"; +import React from "react"; +import {useStoreBucket} from "../../store/bucket"; +import {Bucket, S3File} from "../../interfaces/connection"; +import {useStoreFile} from "../../store/file"; +import {useStoreConnection} from "../../store/connection"; + +const useStyles = makeStyles({ + container: { + marginTop: '0.5rem', + maxWidth: 'calc(100vw - 25rem - 1px)', + }, + row: { + height: '32px', + display: 'flex', + marginLeft: '0.5rem', + marginRight: '0.5rem', + }, + item: { + width: '100%', + maxWidth: '100%', + "&:hover": { + color: tokens.colorNeutralForeground2BrandPressed, + } + }, + text: { + overflow: 'hidden', + width: 'calc(100vw - 32rem)', + display: "block", + } +}) + +export function ListFileComponent() { + + const styles = useStyles(); + const {conn_active} = useStoreConnection(); + const {bucket_active} = useStoreBucket() + const {prefix, files_get, files_list} = useStoreFile() + + async function handleClick(item: S3File) { + if (item.type === 1) { + console.log(`[DEBUG] click prefix = ${prefix} item.key = ${item.key}`) + files_get(conn_active!, bucket_active!, prefix + item.key) + return + } + } + + function handleRightClick(e: React.MouseEvent, item: S3File) { + e.preventDefault() + } + + return + + {(idx) => { + return
{ + await handleClick(files_list[idx]) + }} + onContextMenu={async (e) => { + handleRightClick(e, files_list[idx]) + }}> + : }> + + {files_list[idx].name} + + +
+ }} + + +} \ No newline at end of file diff --git a/frontend/src/component/file/path.tsx b/frontend/src/component/file/path.tsx index d627188..c4e1308 100644 --- a/frontend/src/component/file/path.tsx +++ b/frontend/src/component/file/path.tsx @@ -1,14 +1,92 @@ -import {makeStyles} from "@fluentui/react-components"; +import {Button, Input, makeStyles, Text, tokens, Tooltip} from "@fluentui/react-components"; +import {useStoreBucket} from "../../store/bucket"; +import {ArchiveRegular, ArrowCurveUpLeftFilled} from "@fluentui/react-icons"; +import {useStoreFile} from "../../store/file"; +import React, {ChangeEvent, useState} from "react"; +import {debounce} from 'lodash' +import {useStoreConnection} from "../../store/connection"; const useStyles = makeStyles({ - path: { + container: { height: '4rem', width: '100%', borderBottom: '1px solid lightgray', + display: 'flex', + alignItems: 'center', + }, + show: { + marginLeft: '0.5rem', + height: '100%', + display: 'flex', + alignItems: 'center', + }, + show_text: { + backgroundColor: tokens.colorNeutralBackground1Hover, + padding: '0.5rem 0.5rem', + borderRadius: '0.5rem', + cursor: 'pointer', + display: 'flex', + alignItems: 'center', + marginLeft: '0.5rem', + '&:hover': { + textDecoration: 'none', + backgroundColor: tokens.colorNeutralBackground1Pressed, + }, + }, + op_up: {}, + filter_prefix: { + margin: '0.5rem', }, }) export function Path() { const styles = useStyles() - return
-} \ No newline at end of file + const {conn_active} = useStoreConnection() + const {bucket_active} = useStoreBucket() + const {prefix, files_get} = useStoreFile() + + async function handleClickUp() { + + } + + + const handleFilterChange = debounce((e) => { + console.log('[DEBUG] e =', e) + files_get(conn_active!, bucket_active!, prefix + e.target.value) + }, 500) + + return
+ {bucket_active && ( + <> +
+ +
+
+ { + handleFilterChange(e) + }} + placeholder={"输入前缀过滤"} + contentBefore={/} + /> +
+ + )} +
+} diff --git a/frontend/src/interfaces/connection.ts b/frontend/src/interfaces/connection.ts index a72ff9a..8dd2210 100644 --- a/frontend/src/interfaces/connection.ts +++ b/frontend/src/interfaces/connection.ts @@ -12,6 +12,11 @@ export interface Bucket { name: string; created_at: number; } + export interface S3File { - name:string; + name: string; + key: string; + last_modified: number; + size: number; + type: 0 | 1; } \ No newline at end of file diff --git a/frontend/src/store/bucket.tsx b/frontend/src/store/bucket.tsx index c165115..dadd765 100644 --- a/frontend/src/store/bucket.tsx +++ b/frontend/src/store/bucket.tsx @@ -3,6 +3,8 @@ import {Bucket, Connection} from "../interfaces/connection"; import {Dial, Resp} from "../api"; interface StoreBucket { + bucket_active: Bucket | null; + bucket_set: (Bucket: Bucket | null) => void; bucket_list: Bucket[]; bucket_get: (conn: Connection, refresh: boolean) => void; } @@ -10,23 +12,27 @@ interface StoreBucket { let bucket_map: { [id: number]: Bucket[] }; export const useStoreBucket = create()((set) => ({ + bucket_active: null, + bucket_set: async (bucket: Bucket | null) => { + set({bucket_active: bucket}); + }, bucket_list: [], bucket_get: async (conn: Connection, refresh: boolean) => { let res: Resp<{ list: Bucket[]; }>; if (refresh) { - res = await Dial<{list: Bucket[]}>('/api/connection/buckets', {id: conn.id}); + res = await Dial<{ list: Bucket[] }>('/api/connection/buckets', {id: conn.id}); if (res.status !== 200) { return } } set((state) => { - if (refresh) { - bucket_map = {...bucket_map, [conn.id]: res.data.list} - return {bucket_list: res.data.list}; - } + if (refresh) { + bucket_map = {...bucket_map, [conn.id]: res.data.list} + return {bucket_list: res.data.list}; + } - return {bucket_list: bucket_map[conn.id]}; + return {bucket_list: bucket_map[conn.id]}; }) } })) diff --git a/frontend/src/store/file.tsx b/frontend/src/store/file.tsx index 58e1e06..3014544 100644 --- a/frontend/src/store/file.tsx +++ b/frontend/src/store/file.tsx @@ -1,12 +1,29 @@ import {create} from 'zustand' -import {Bucket, S3File} from "../interfaces/connection"; +import {Bucket, Connection, S3File} from "../interfaces/connection"; +import {Dial, Resp} from "../api"; interface StoreFile { - bucket: Bucket; - file_list: S3File[]; + prefix: string; + files_list: S3File[]; + files_get: (conn: Connection, bucket: Bucket, prefix: string) => void; } export const useStoreFile = create()((set) => ({ - bucket: {name: '', created_at: 0}, - file_list: [], + prefix: "", + files_list: [], + files_get: async (conn: Connection, bucket: Bucket, prefix = '') => { + const res = await Dial<{ list: S3File[] }>('/api/bucket/files', { + conn_id: conn.id, + bucket: bucket.name, + prefix: prefix , + }) + + if (res.status !== 200) { + return + } + + set(state => { + return {prefix: prefix, files_list: res.data.list} + }) + } })) diff --git a/internal/api/api.go b/internal/api/api.go index 9b42b4c..510a5f8 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -30,7 +30,7 @@ func Init(ctx context.Context) error { register("/api/connection/connect", handler.ConnectionConnect) register("/api/connection/disconnect", handler.ConnectionDisconnect) register("/api/connection/buckets", handler.ConnectionBuckets) - register("/api/bucket/file", handler.BucketFile) + register("/api/bucket/files", handler.BucketFile) return nil } diff --git a/internal/handler/bucket.go b/internal/handler/bucket.go index 3f55a1b..23b579a 100644 --- a/internal/handler/bucket.go +++ b/internal/handler/bucket.go @@ -8,15 +8,16 @@ import ( func BucketFile(c *ndh.Ctx) error { type Req struct { - ConnId uint64 `json:"conn_id"` - Bucket string `json:"bucket"` - Keyword string `json:"keyword"` + ConnId uint64 `json:"conn_id"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` } var ( err error req = new(Req) client *s3.Client + list []*s3.ListFileRes ) if err = c.ReqParse(req); err != nil { @@ -31,5 +32,9 @@ func BucketFile(c *ndh.Ctx) error { return c.Send500(err.Error()) } - client.ListFile() + if list, err = client.ListFile(c.Context(), req.Bucket, req.Prefix); err != nil { + return c.Send500(err.Error()) + } + + return c.Send200(map[string]any{"list": list}) } diff --git a/internal/s3/list.go b/internal/s3/list.go index 55bddbd..7df9965 100644 --- a/internal/s3/list.go +++ b/internal/s3/list.go @@ -15,10 +15,19 @@ type ListBucketRes struct { Name string `json:"name"` } +type ListFileType int64 + +const ( + ListFileTypeFile ListFileType = iota + ListFileTypeDir +) + type ListFileRes struct { - Name string - LastModified time.Time - Size int64 + Name string `json:"name"` + Key string `json:"key"` + LastModified time.Time `json:"last_modified"` + Size int64 `json:"size"` + Type ListFileType `json:"type"` } func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) { @@ -44,7 +53,7 @@ func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) { return res, nil } -func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, parent string) ([]*ListFileRes, error) { +func (c *Client) ListFile(ctx context.Context, bucket string, prefix string) ([]*ListFileRes, error) { var ( err error input = &s3.ListObjectsV2Input{ @@ -66,9 +75,11 @@ func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, par folder := lo.FilterMap( output.CommonPrefixes, func(item types.CommonPrefix, index int) (*ListFileRes, bool) { - name := strings.TrimPrefix(*item.Prefix, parent) + name := strings.TrimPrefix(*item.Prefix, prefix) return &ListFileRes{ Name: name, + Key: name, + Type: ListFileTypeDir, }, name != "" }, ) @@ -77,9 +88,11 @@ func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, par output.Contents, func(item types.Object, index int) *ListFileRes { return &ListFileRes{ - Name: *item.Key, + Key: *item.Key, + Name: strings.TrimPrefix(*item.Key, prefix), LastModified: *item.LastModified, Size: *item.Size, + Type: ListFileTypeFile, } }, ) diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index 8b6aabe..583d300 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -24,7 +24,7 @@ func TestListFile(t *testing.T) { t.Fatalf("call s3.New err = %s", err.Error()) } - files, err := cli.ListFile(tool.Timeout(30), "infobox-person", "") + files, err := cli.ListFile(tool.Timeout(30), "topic-audit", "") if err != nil { t.Fatalf("call s3.ListFile err = %s", err.Error()) } @@ -32,6 +32,6 @@ func TestListFile(t *testing.T) { t.Logf("[x] file length = %d", len(files)) for _, item := range files { - t.Logf("[x] file = %s, size = %d", item.Name, item.Size) + t.Logf("[x: %d] file = %s, size = %d", item.Type, item.Name, item.Size) } }