feat: add database.s3 support
This commit is contained in:
loveuer
parent
3c0b7479a0
commit
534b9586f2
20
go.mod
20
go.mod
@ -1,11 +1,15 @@
|
||||
module ultone
|
||||
|
||||
go 1.20
|
||||
go 1.21
|
||||
|
||||
toolchain go1.23.4
|
||||
|
||||
require (
|
||||
gitea.com/taozitaozi/gredis v0.0.0-20241226104049-af698e5ad477
|
||||
github.com/aws/aws-sdk-go-v2 v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.28.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.48
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0
|
||||
github.com/elastic/go-elasticsearch/v7 v7.17.10
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/golang-jwt/jwt/v5 v5.2.0
|
||||
@ -32,6 +36,20 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect
|
||||
github.com/aws/smithy-go v1.22.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/glebarez/go-sqlite v1.21.2 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
|
||||
50
go.sum
50
go.sum
@ -4,6 +4,42 @@ gitea.com/taozitaozi/gredis v0.0.0-20241226104049-af698e5ad477/go.mod h1:E5Y+tGZ
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc=
|
||||
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
|
||||
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@ -33,8 +69,11 @@ github.com/facebook/fbthrift v0.31.1-0.20211129061412-801ed7f9f295/go.mod h1:2tn
|
||||
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
|
||||
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
|
||||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
|
||||
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
|
||||
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
|
||||
@ -73,6 +112,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@ -144,10 +184,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
@ -174,10 +216,13 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
|
||||
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@ -195,6 +240,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
|
||||
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
|
||||
@ -236,6 +282,7 @@ github.com/tdewolff/parse/v2 v2.7.11 h1:v+W45LnzmjndVlfqPCT5gGjAAZKd1GJGOPJveTIk
|
||||
github.com/tdewolff/parse/v2 v2.7.11/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
|
||||
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
|
||||
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
|
||||
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
|
||||
github.com/vesoft-inc/nebula-go/v3 v3.5.0 h1:2ZSkoBxtIfs15AXJXqrAPDPd0Z9HrzKR7YKXPqlJcR0=
|
||||
github.com/vesoft-inc/nebula-go/v3 v3.5.0/go.mod h1:+sXv05jYQBARdTbTcIEsWVXCnF/6ttOlDK35xQ6m54s=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@ -252,6 +299,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
@ -408,8 +456,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
193
internal/database/s3/by_dir.go
Normal file
193
internal/database/s3/by_dir.go
Normal file
@ -0,0 +1,193 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"ultone/internal/log"
|
||||
|
||||
"github.com/samber/lo"
|
||||
)
|
||||
|
||||
type s3_dir struct {
|
||||
ctx context.Context
|
||||
dir string
|
||||
}
|
||||
|
||||
func (s *s3_dir) clean() {
|
||||
now := time.Now()
|
||||
|
||||
filepath.Walk(s.dir, func(p string, info fs.FileInfo, err error) error {
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(info.Name(), ".meta_") {
|
||||
bs, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
log.Warn(s.ctx, "s3.by_dir: read meta err, err = %s", err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
nm := new(meta)
|
||||
if err = json.Unmarshal(bs, nm); err != nil {
|
||||
log.Warn(s.ctx, "s3.by_dir: unmarshal meta err, err = %s", err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
if nm.ExpireAt > 0 && time.UnixMilli(nm.ExpireAt).Before(now) {
|
||||
os.Remove(p)
|
||||
base := strings.TrimPrefix(info.Name(), ".meta_")
|
||||
dir := filepath.Dir(p)
|
||||
os.Remove(path.Join(dir, base))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Delete implements S3.
|
||||
func (s *s3_dir) Delete(ctx context.Context, bucket string, key string) error {
|
||||
location := path.Join(s.dir, bucket, key)
|
||||
info, err := os.Stat(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("target is dir")
|
||||
}
|
||||
|
||||
os.Remove(path.Join(s.dir, bucket, ".meta_"+key))
|
||||
|
||||
return os.Remove(location)
|
||||
}
|
||||
|
||||
// Get implements S3.
|
||||
func (s *s3_dir) Get(ctx context.Context, bucket string, key string) (*Object, error) {
|
||||
location := path.Join(s.dir, bucket, key)
|
||||
info, err := os.Stat(location)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("target is dir")
|
||||
}
|
||||
|
||||
var (
|
||||
f io.ReadCloser
|
||||
bs []byte
|
||||
obj = &Object{}
|
||||
)
|
||||
|
||||
if f, err = os.Open(location); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj.Body = f
|
||||
|
||||
if bs, err = os.ReadFile(path.Join(s.dir, bucket, ".meta_"+key)); err != nil {
|
||||
log.Warn(ctx, "s3.dir: open file meta err, err = %v", err)
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
m := new(meta)
|
||||
if err = json.Unmarshal(bs, m); err != nil {
|
||||
log.Warn(ctx, "s3.dir: unmarshal meta err, err = %v", err)
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
obj.ContentType = m.ContentType
|
||||
obj.Size = m.Size
|
||||
obj.ExpireAt = m.ExpireAt
|
||||
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// Put implements S3.
|
||||
func (s *s3_dir) Put(ctx context.Context, bucket string, key string, obj *Object) error {
|
||||
if bucket != "" {
|
||||
os.MkdirAll(path.Join(s.dir, bucket), 0o755)
|
||||
}
|
||||
|
||||
location := path.Join(s.dir, bucket, key)
|
||||
ml := path.Join(s.dir, bucket, ".meta_"+key)
|
||||
|
||||
bs, err := io.ReadAll(obj.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if obj.Size != 0 && obj.Size != int64(len(bs)) {
|
||||
return fmt.Errorf("object size mismatch")
|
||||
}
|
||||
|
||||
obj.Size = int64(len(bs))
|
||||
|
||||
if obj.ContentType == "" {
|
||||
obj.ContentType = http.DetectContentType(lo.If(len(bs) >= 128, bs[:128]).Else(bs))
|
||||
}
|
||||
|
||||
if err = os.WriteFile(location, bs, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := &meta{
|
||||
ContentType: obj.ContentType,
|
||||
Size: obj.Size,
|
||||
ExpireAt: obj.ExpireAt,
|
||||
}
|
||||
|
||||
ms, _ := json.Marshal(m)
|
||||
os.WriteFile(ml, ms, 0o644)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newDirClient(ctx context.Context, dir string) (S3, error) {
|
||||
dir = filepath.ToSlash(dir)
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if err = os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("target dir exist but not dir")
|
||||
}
|
||||
|
||||
c := &s3_dir{ctx: ctx, dir: dir}
|
||||
|
||||
// todo: expire files
|
||||
go func() {
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case t := <-ticker.C:
|
||||
log.Debug(ctx, "s3.by_dir: start clean up s3 dir @%s", t.Format("2006-01-02T15:04:05"))
|
||||
c.clean()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return c, nil
|
||||
}
|
||||
112
internal/database/s3/by_s3.go
Normal file
112
internal/database/s3/by_s3.go
Normal file
@ -0,0 +1,112 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"ultone/internal/log"
|
||||
"ultone/internal/tool"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
type s3_client struct {
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
// Delete implements S3.
|
||||
func (s *s3_client) Delete(ctx context.Context, bucket string, key string) error {
|
||||
var err error
|
||||
|
||||
if _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get implements S3.
|
||||
func (s *s3_client) Get(ctx context.Context, bucket string, key string) (*Object, error) {
|
||||
var (
|
||||
err error
|
||||
res *s3.GetObjectOutput
|
||||
)
|
||||
|
||||
if res, err = s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
ContentType: *res.ContentType,
|
||||
Body: res.Body,
|
||||
ExpireAt: res.Expires.UnixMilli(),
|
||||
Size: *res.ContentLength,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put implements S3.
|
||||
func (s *s3_client) Put(ctx context.Context, bucket string, key string, obj *Object) error {
|
||||
var err error
|
||||
|
||||
if _, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
ACL: types.ObjectCannedACLPublicRead,
|
||||
Body: obj.Body,
|
||||
ContentType: aws.String(obj.ContentType),
|
||||
Expires: aws.Time(time.UnixMilli(obj.ExpireAt)),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newS3Client(ctx context.Context, endpoint, access, key string) (S3, error) {
|
||||
var (
|
||||
err error
|
||||
sdkConfig aws.Config
|
||||
output *s3.ListBucketsOutput
|
||||
)
|
||||
|
||||
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
}, nil
|
||||
})
|
||||
|
||||
if sdkConfig, err = config.LoadDefaultConfig(
|
||||
ctx,
|
||||
config.WithEndpointResolverWithOptions(customResolver),
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) {
|
||||
// o.BaseEndpoint = aws.String(endpoint)
|
||||
// o.EndpointResolverV2 = &resolverV2{}
|
||||
o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(access, key, ""))
|
||||
o.UsePathStyle = true
|
||||
o.Region = "auto"
|
||||
})
|
||||
|
||||
if output, err = s3Client.ListBuckets(tool.Timeout(5), &s3.ListBucketsInput{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range output.Buckets {
|
||||
log.Debug(ctx, "s3.New: list bucket name = %s", *item.Name)
|
||||
}
|
||||
|
||||
return &s3_client{client: s3Client}, nil
|
||||
}
|
||||
35
internal/database/s3/new.go
Normal file
35
internal/database/s3/new.go
Normal file
@ -0,0 +1,35 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
var Default S3
|
||||
|
||||
func New(ctx context.Context, uri string) (S3, error) {
|
||||
ins, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch ins.Scheme {
|
||||
case "http", "https":
|
||||
if ins.User == nil {
|
||||
return nil, fmt.Errorf("missing access or key")
|
||||
}
|
||||
access := ins.User.Username()
|
||||
key, _ := ins.User.Password()
|
||||
return newS3Client(ctx, fmt.Sprintf("%s://%s", ins.Scheme, ins.Host), access, key)
|
||||
case "dir":
|
||||
return newDirClient(ctx, ins.Host)
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid new s3 uri scheme: %s", ins.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func Init(ctx context.Context, uri string) (err error) {
|
||||
Default, err = New(ctx, uri)
|
||||
return err
|
||||
}
|
||||
25
internal/database/s3/s3.go
Normal file
25
internal/database/s3/s3.go
Normal file
@ -0,0 +1,25 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
type meta struct {
|
||||
ContentType string `json:"content_type"`
|
||||
Size int64 `json:"size"`
|
||||
ExpireAt int64 `json:"expire_at"`
|
||||
}
|
||||
|
||||
type Object struct {
|
||||
ContentType string
|
||||
Body io.ReadCloser
|
||||
Size int64
|
||||
ExpireAt int64
|
||||
}
|
||||
|
||||
type S3 interface {
|
||||
Get(ctx context.Context, bucket, key string) (*Object, error)
|
||||
Put(ctx context.Context, bucket, key string, obj *Object) error
|
||||
Delete(ctx context.Context, bucket, key string) error
|
||||
}
|
||||
@ -22,3 +22,29 @@ func HumanDuration(nano int64) string {
|
||||
|
||||
return fmt.Sprintf("%6.2f%s", duration, unit)
|
||||
}
|
||||
|
||||
func HumanSize(size int64) string {
|
||||
const (
|
||||
_ = iota
|
||||
KB = 1 << (10 * iota) // 1 KB = 1024 bytes
|
||||
MB // 1 MB = 1024 KB
|
||||
GB // 1 GB = 1024 MB
|
||||
TB // 1 TB = 1024 GB
|
||||
PB // 1 PB = 1024 TB
|
||||
)
|
||||
|
||||
switch {
|
||||
case size >= PB:
|
||||
return fmt.Sprintf("%.2f PB", float64(size)/PB)
|
||||
case size >= TB:
|
||||
return fmt.Sprintf("%.2f TB", float64(size)/TB)
|
||||
case size >= GB:
|
||||
return fmt.Sprintf("%.2f GB", float64(size)/GB)
|
||||
case size >= MB:
|
||||
return fmt.Sprintf("%.2f MB", float64(size)/MB)
|
||||
case size >= KB:
|
||||
return fmt.Sprintf("%.2f KB", float64(size)/KB)
|
||||
default:
|
||||
return fmt.Sprintf("%d bytes", size)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user