loveuer/nf-disk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: s3 file prefix filter

Browse Source
This commit is contained in:
zhaoyupeng 2024-10-11 18:03:09 +08:00
parent 8bc2a2541d
commit f54ed67f0f
14 changed files with 286 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
frontend/package-lock.json generated
View File

@ -15,6 +15,7 @@
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.26.2",
"use-debounce": "^10.0.3",
"zustand": "^5.0.0-rc.2"
},
"devDependencies": {
@ -3465,6 +3466,18 @@
"browserslist": ">= 4.21.0"
}
},
"node_modules/use-debounce": {
"version": "10.0.3",
"resolved": "https://registry.npmmirror.com/use-debounce/-/use-debounce-10.0.3.tgz",
"integrity": "sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==",
"license": "MIT",
"engines": {
"node": ">= 16.0.0"
},
"peerDependencies": {
"react": "*"
}
},
"node_modules/use-disposable": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/use-disposable/-/use-disposable-1.0.2.tgz",
@ -5915,6 +5928,12 @@
"picocolors": "^1.0.1"
}
},
"use-debounce": {
"version": "10.0.3",
"resolved": "https://registry.npmmirror.com/use-debounce/-/use-debounce-10.0.3.tgz",
"integrity": "sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==",
"requires": {}
},
"use-disposable": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/use-disposable/-/use-disposable-1.0.2.tgz",

1
frontend/package.json
View File

@ -16,6 +16,7 @@
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.26.2",
"use-debounce": "^10.0.3",
"zustand": "^5.0.0-rc.2"
},
"devDependencies": {

13
frontend/src/component/connection/list.tsx
View File

@ -11,7 +11,7 @@ import {
} from "@fluentui/react-components"
import {DatabaseLinkRegular, DismissRegular} from "@fluentui/react-icons";
import React, {useState} from "react";
import {Connection} from "../../interfaces/connection";
import {Bucket, Connection} from "../../interfaces/connection";
import {useToast} from "../../message";
import {Dial} from "../../api";
import {useStoreConnection} from "../../store/connection";
@ -77,10 +77,16 @@ export function ConnectionList() {
const {dispatchMessage} = useToast();
const {conn_list, conn_update} = useStoreConnection();
const [conn_filter, set_conn_filter] = useState<string>('');
const {bucket_get} = useStoreBucket()
const {bucket_get, bucket_set} = useStoreBucket()
async function handleSelect(item: Connection) {
conn_list.map((one: Connection) => {
if (item.id === one.id && one.active) {
conn_update(one)
bucket_get(one, false)
bucket_set(null)
}
})
}
async function handleConnect(item: Connection) {
@ -92,6 +98,7 @@ export function ConnectionList() {
conn_update({...item, active: true})
bucket_get(item, true)
bucket_set(null)
}
async function handleDisconnect(item: Connection) {

13
frontend/src/component/file/content.tsx
View File

@ -1,8 +1,9 @@
import {Path} from "./path";
import {ListComponent} from "./list";
import {ListBucketComponent} from "./list_bucket";
import {makeStyles} from "@fluentui/react-components";
import {useStoreBucket} from "../../store/bucket";
import {useStoreFile} from "../../store/file";
import {ListFileComponent} from "./list_file";
const useStyles = makeStyles({
content: {
@ -17,15 +18,15 @@ const useStyles = makeStyles({
export function Content() {
const styles = useStyles()
const {bucket_list} = useStoreBucket()
const {bucket, file_list} = useStoreFile()
const {bucket_active, bucket_list} = useStoreBucket()
const {file_list} = useStoreFile()
return <div className={styles.content}>
<Path/>
{
bucket.name ?
<ListComponent type={'file'} list={file_list.map(item => item.name)}/> :
<ListComponent type={'bucket'} list={bucket_list.map(item => item.name)}/>
bucket_active ?
<ListFileComponent/> :
<ListBucketComponent/>
}
</div>
}

31
frontend/src/component/file/list.tsx → frontend/src/component/file/list_bucket.tsx
View File

@ -2,6 +2,10 @@ import {makeStyles, MenuItem, MenuList, Text, tokens} from "@fluentui/react-comp
import {ArchiveRegular, DocumentBulletListRegular} from "@fluentui/react-icons";
import {VirtualizerScrollView} from "@fluentui/react-components/unstable";
import React from "react";
import {useStoreBucket} from "../../store/bucket";
import {Bucket} from "../../interfaces/connection";
import {useStoreFile} from "../../store/file";
import {useStoreConnection} from "../../store/connection";
const useStyles = makeStyles({
container: {
@ -28,26 +32,25 @@ const useStyles = makeStyles({
}
})
export interface ListComponentProps {
type: "bucket" | "file"
list: string[],
}
export function ListComponent(props: ListComponentProps) {
export function ListBucketComponent() {
const styles = useStyles();
const {conn_active} = useStoreConnection()
const {bucket_set, bucket_list} = useStoreBucket()
const {files_get} = useStoreFile()
async function handleClick(item: string) {
console.log('[DEBUG] bucket click =', item);
async function handleClick(item: Bucket) {
bucket_set(item)
files_get(conn_active!, item, "")
}
function handleRightClick(e: React.MouseEvent<HTMLDivElement>, string: string) {
function handleRightClick(e: React.MouseEvent<HTMLDivElement>, item: Bucket) {
e.preventDefault()
}
return <MenuList className={styles.container}>
<VirtualizerScrollView
numItems={props.list.length}
numItems={bucket_list.length}
itemSize={32}
container={{role: 'list', style: {maxHeight: 'calc(100vh - 9rem)'}}}
>
@ -55,15 +58,15 @@ export function ListComponent(props: ListComponentProps) {
return <div
className={styles.row} key={idx}
onClick={async () => {
await handleClick(props.list[idx])
await handleClick(bucket_list[idx])
}}
onContextMenu={async (e) => {
handleRightClick(e, props.list[idx])
handleRightClick(e, bucket_list[idx])
}}>
<MenuItem className={styles.item}
icon={props.type === 'bucket' ? <ArchiveRegular/> : <DocumentBulletListRegular/>}>
icon={<ArchiveRegular/>}>
<Text truncate wrap={false} className={styles.text}>
{props.list[idx]}
{bucket_list[idx].name}
</Text>
</MenuItem>
</div>

79
frontend/src/component/file/list_file.tsx Normal file
View File

@ -0,0 +1,79 @@
import {makeStyles, MenuItem, MenuList, Text, tokens} from "@fluentui/react-components";
import {ArchiveRegular, DocumentBulletListRegular, FolderRegular} from "@fluentui/react-icons";
import {VirtualizerScrollView} from "@fluentui/react-components/unstable";
import React from "react";
import {useStoreBucket} from "../../store/bucket";
import {Bucket, S3File} from "../../interfaces/connection";
import {useStoreFile} from "../../store/file";
import {useStoreConnection} from "../../store/connection";
const useStyles = makeStyles({
container: {
marginTop: '0.5rem',
maxWidth: 'calc(100vw - 25rem - 1px)',
},
row: {
height: '32px',
display: 'flex',
marginLeft: '0.5rem',
marginRight: '0.5rem',
},
item: {
width: '100%',
maxWidth: '100%',
"&:hover": {
color: tokens.colorNeutralForeground2BrandPressed,
}
},
text: {
overflow: 'hidden',
width: 'calc(100vw - 32rem)',
display: "block",
}
})
export function ListFileComponent() {
const styles = useStyles();
const {conn_active} = useStoreConnection();
const {bucket_active} = useStoreBucket()
const {prefix, files_get, files_list} = useStoreFile()
async function handleClick(item: S3File) {
if (item.type === 1) {
console.log(`[DEBUG] click prefix = ${prefix} item.key = ${item.key}`)
files_get(conn_active!, bucket_active!, prefix + item.key)
return
}
}
function handleRightClick(e: React.MouseEvent<HTMLDivElement>, item: S3File) {
e.preventDefault()
}
return <MenuList className={styles.container}>
<VirtualizerScrollView
numItems={files_list.length}
itemSize={32}
container={{role: 'list', style: {maxHeight: 'calc(100vh - 9rem)'}}}
>
{(idx) => {
return <div
className={styles.row} key={idx}
onClick={async () => {
await handleClick(files_list[idx])
}}
onContextMenu={async (e) => {
handleRightClick(e, files_list[idx])
}}>
<MenuItem className={styles.item}
icon={files_list[idx].type ? <FolderRegular/> : <DocumentBulletListRegular/>}>
<Text truncate wrap={false} className={styles.text}>
{files_list[idx].name}
</Text>
</MenuItem>
</div>
}}
</VirtualizerScrollView>
</MenuList>
}

84
frontend/src/component/file/path.tsx
View File

@ -1,14 +1,92 @@
import {makeStyles} from "@fluentui/react-components";
import {Button, Input, makeStyles, Text, tokens, Tooltip} from "@fluentui/react-components";
import {useStoreBucket} from "../../store/bucket";
import {ArchiveRegular, ArrowCurveUpLeftFilled} from "@fluentui/react-icons";
import {useStoreFile} from "../../store/file";
import React, {ChangeEvent, useState} from "react";
import {debounce} from 'lodash'
import {useStoreConnection} from "../../store/connection";
const useStyles = makeStyles({
path: {
container: {
height: '4rem',
width: '100%',
borderBottom: '1px solid lightgray',
display: 'flex',
alignItems: 'center',
},
show: {
marginLeft: '0.5rem',
height: '100%',
display: 'flex',
alignItems: 'center',
},
show_text: {
backgroundColor: tokens.colorNeutralBackground1Hover,
padding: '0.5rem 0.5rem',
borderRadius: '0.5rem',
cursor: 'pointer',
display: 'flex',
alignItems: 'center',
marginLeft: '0.5rem',
'&:hover': {
textDecoration: 'none',
backgroundColor: tokens.colorNeutralBackground1Pressed,
},
},
op_up: {},
filter_prefix: {
margin: '0.5rem',
},
})
export function Path() {
const styles = useStyles()
return <div className={styles.path}></div>
const {conn_active} = useStoreConnection()
const {bucket_active} = useStoreBucket()
const {prefix, files_get} = useStoreFile()
async function handleClickUp() {
}
const handleFilterChange = debounce((e) => {
console.log('[DEBUG] e =', e)
files_get(conn_active!, bucket_active!, prefix + e.target.value)
}, 500)
return <div className={styles.container}>
{bucket_active && (
<>
<div className={styles.show}>
<Tooltip content="返回上一级" relationship="label">
<Button className={styles.op_up}
onClick={async () => {
await handleClickUp()
}}
size="small" icon={<ArrowCurveUpLeftFilled/>}/>
</Tooltip>
<Text className={styles.show_text}><ArchiveRegular
style={{marginRight: '0.5rem'}}/>{bucket_active.name}</Text>
{prefix && (
prefix.split("/").filter(item => item).map(item => {
return <>
<Text style={{marginLeft: '0.5rem'}}>/</Text>
<Text className={styles.show_text}>{item}</Text>
</>
})
)}
</div>
<div className={styles.filter_prefix}>
<Input
onChange={(e) => {
handleFilterChange(e)
}}
placeholder={"输入前缀过滤"}
contentBefore={<Text>/</Text>}
/>
</div>
</>
)}
</div>
}

7
frontend/src/interfaces/connection.ts
View File

@ -12,6 +12,11 @@ export interface Bucket {
name: string;
created_at: number;
}
export interface S3File {
name:string;
name: string;
key: string;
last_modified: number;
size: number;
type: 0 | 1;
}

8
frontend/src/store/bucket.tsx
View File

@ -3,6 +3,8 @@ import {Bucket, Connection} from "../interfaces/connection";
import {Dial, Resp} from "../api";
interface StoreBucket {
bucket_active: Bucket | null;
bucket_set: (Bucket: Bucket | null) => void;
bucket_list: Bucket[];
bucket_get: (conn: Connection, refresh: boolean) => void;
}
@ -10,11 +12,15 @@ interface StoreBucket {
let bucket_map: { [id: number]: Bucket[] };
export const useStoreBucket = create<StoreBucket>()((set) => ({
bucket_active: null,
bucket_set: async (bucket: Bucket | null) => {
set({bucket_active: bucket});
},
bucket_list: [],
bucket_get: async (conn: Connection, refresh: boolean) => {
let res: Resp<{ list: Bucket[]; }>;
if (refresh) {
res = await Dial<{list: Bucket[]}>('/api/connection/buckets', {id: conn.id});
res = await Dial<{ list: Bucket[] }>('/api/connection/buckets', {id: conn.id});
if (res.status !== 200) {
return
}

27
frontend/src/store/file.tsx
View File

@ -1,12 +1,29 @@
import {create} from 'zustand'
import {Bucket, S3File} from "../interfaces/connection";
import {Bucket, Connection, S3File} from "../interfaces/connection";
import {Dial, Resp} from "../api";
interface StoreFile {
bucket: Bucket;
file_list: S3File[];
prefix: string;
files_list: S3File[];
files_get: (conn: Connection, bucket: Bucket, prefix: string) => void;
}
export const useStoreFile = create<StoreFile>()((set) => ({
bucket: {name: '', created_at: 0},
file_list: [],
prefix: "",
files_list: [],
files_get: async (conn: Connection, bucket: Bucket, prefix = '') => {
const res = await Dial<{ list: S3File[] }>('/api/bucket/files', {
conn_id: conn.id,
bucket: bucket.name,
prefix: prefix ,
})
if (res.status !== 200) {
return
}
set(state => {
return {prefix: prefix, files_list: res.data.list}
})
}
}))

2
internal/api/api.go
View File

@ -30,7 +30,7 @@ func Init(ctx context.Context) error {
register("/api/connection/connect", handler.ConnectionConnect)
register("/api/connection/disconnect", handler.ConnectionDisconnect)
register("/api/connection/buckets", handler.ConnectionBuckets)
register("/api/bucket/file", handler.BucketFile)
register("/api/bucket/files", handler.BucketFile)
return nil
}

9
internal/handler/bucket.go
View File

@ -10,13 +10,14 @@ func BucketFile(c *ndh.Ctx) error {
type Req struct {
ConnId uint64 `json:"conn_id"`
Bucket string `json:"bucket"`
Keyword string `json:"keyword"`
Prefix string `json:"prefix"`
}
var (
err error
req = new(Req)
client *s3.Client
list []*s3.ListFileRes
)
if err = c.ReqParse(req); err != nil {
@ -31,5 +32,9 @@ func BucketFile(c *ndh.Ctx) error {
return c.Send500(err.Error())
}
client.ListFile()
if list, err = client.ListFile(c.Context(), req.Bucket, req.Prefix); err != nil {
return c.Send500(err.Error())
}
return c.Send200(map[string]any{"list": list})
}

25
internal/s3/list.go
View File

@ -15,10 +15,19 @@ type ListBucketRes struct {
Name string `json:"name"`
}
type ListFileType int64
const (
ListFileTypeFile ListFileType = iota
ListFileTypeDir
)
type ListFileRes struct {
Name string
LastModified time.Time
Size int64
Name string `json:"name"`
Key string `json:"key"`
LastModified time.Time `json:"last_modified"`
Size int64 `json:"size"`
Type ListFileType `json:"type"`
}
func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) {
@ -44,7 +53,7 @@ func (c *Client) ListBucket(ctx context.Context) ([]*ListBucketRes, error) {
return res, nil
}
func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, parent string) ([]*ListFileRes, error) {
func (c *Client) ListFile(ctx context.Context, bucket string, prefix string) ([]*ListFileRes, error) {
var (
err error
input = &s3.ListObjectsV2Input{
@ -66,9 +75,11 @@ func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, par
folder := lo.FilterMap(
output.CommonPrefixes,
func(item types.CommonPrefix, index int) (*ListFileRes, bool) {
name := strings.TrimPrefix(*item.Prefix, parent)
name := strings.TrimPrefix(*item.Prefix, prefix)
return &ListFileRes{
Name: name,
Key: name,
Type: ListFileTypeDir,
}, name != ""
},
)
@ -77,9 +88,11 @@ func (c *Client) ListFile(ctx context.Context, bucket string, prefix string, par
output.Contents,
func(item types.Object, index int) *ListFileRes {
return &ListFileRes{
Name: *item.Key,
Key: *item.Key,
Name: strings.TrimPrefix(*item.Key, prefix),
LastModified: *item.LastModified,
Size: *item.Size,
Type: ListFileTypeFile,
}
},
)

4
internal/s3/s3_test.go
View File

@ -24,7 +24,7 @@ func TestListFile(t *testing.T) {
t.Fatalf("call s3.New err = %s", err.Error())
}
files, err := cli.ListFile(tool.Timeout(30), "infobox-person", "")
files, err := cli.ListFile(tool.Timeout(30), "topic-audit", "")
if err != nil {
t.Fatalf("call s3.ListFile err = %s", err.Error())
}
@ -32,6 +32,6 @@ func TestListFile(t *testing.T) {
t.Logf("[x] file length = %d", len(files))
for _, item := range files {
t.Logf("[x] file = %s, size = %d", item.Name, item.Size)
t.Logf("[x: %d] file = %s, size = %d", item.Type, item.Name, item.Size)
}
}