From 534b9586f2f1d537da31226f6cd8ca2732cbf060 Mon Sep 17 00:00:00 2001 From: loveuer Date: Mon, 6 Jan 2025 23:38:49 -0800 Subject: [PATCH] feat: add database.s3 support --- go.mod | 20 +++- go.sum | 50 +++++++++ internal/database/s3/by_dir.go | 193 +++++++++++++++++++++++++++++++++ internal/database/s3/by_s3.go | 112 +++++++++++++++++++ internal/database/s3/new.go | 35 ++++++ internal/database/s3/s3.go | 25 +++++ internal/tool/human.go | 26 +++++ 7 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 internal/database/s3/by_dir.go create mode 100644 internal/database/s3/by_s3.go create mode 100644 internal/database/s3/new.go create mode 100644 internal/database/s3/s3.go diff --git a/go.mod b/go.mod index d969e9b..fd26f9c 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,15 @@ module ultone -go 1.20 +go 1.21 toolchain go1.23.4 require ( gitea.com/taozitaozi/gredis v0.0.0-20241226104049-af698e5ad477 + github.com/aws/aws-sdk-go-v2 v1.32.7 + github.com/aws/aws-sdk-go-v2/config v1.28.7 + github.com/aws/aws-sdk-go-v2/credentials v1.17.48 + github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt/v5 v5.2.0 @@ -32,6 +36,20 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/go.sum b/go.sum index 87095eb..312be98 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,42 @@ gitea.com/taozitaozi/gredis v0.0.0-20241226104049-af698e5ad477/go.mod h1:E5Y+tGZ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= +github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= +github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -33,8 +69,11 @@ github.com/facebook/fbthrift v0.31.1-0.20211129061412-801ed7f9f295/go.mod h1:2tn github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= @@ -73,6 +112,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -144,10 +184,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -174,10 +216,13 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -195,6 +240,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -236,6 +282,7 @@ github.com/tdewolff/parse/v2 v2.7.11 h1:v+W45LnzmjndVlfqPCT5gGjAAZKd1GJGOPJveTIk github.com/tdewolff/parse/v2 v2.7.11/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo= +github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/vesoft-inc/nebula-go/v3 v3.5.0 h1:2ZSkoBxtIfs15AXJXqrAPDPd0Z9HrzKR7YKXPqlJcR0= github.com/vesoft-inc/nebula-go/v3 v3.5.0/go.mod h1:+sXv05jYQBARdTbTcIEsWVXCnF/6ttOlDK35xQ6m54s= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -252,6 +299,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -408,8 +456,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/database/s3/by_dir.go b/internal/database/s3/by_dir.go new file mode 100644 index 0000000..00ce9e5 --- /dev/null +++ b/internal/database/s3/by_dir.go @@ -0,0 +1,193 @@ +package s3 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "ultone/internal/log" + + "github.com/samber/lo" +) + +type s3_dir struct { + ctx context.Context + dir string +} + +func (s *s3_dir) clean() { + now := time.Now() + + filepath.Walk(s.dir, func(p string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + if strings.HasPrefix(info.Name(), ".meta_") { + bs, err := os.ReadFile(p) + if err != nil { + log.Warn(s.ctx, "s3.by_dir: read meta err, err = %s", err.Error()) + return nil + } + + nm := new(meta) + if err = json.Unmarshal(bs, nm); err != nil { + log.Warn(s.ctx, "s3.by_dir: unmarshal meta err, err = %s", err.Error()) + return nil + } + + if nm.ExpireAt > 0 && time.UnixMilli(nm.ExpireAt).Before(now) { + os.Remove(p) + base := strings.TrimPrefix(info.Name(), ".meta_") + dir := filepath.Dir(p) + os.Remove(path.Join(dir, base)) + } + } + + return nil + }) +} + +// Delete implements S3. +func (s *s3_dir) Delete(ctx context.Context, bucket string, key string) error { + location := path.Join(s.dir, bucket, key) + info, err := os.Stat(location) + if err != nil { + return err + } + + if info.IsDir() { + return fmt.Errorf("target is dir") + } + + os.Remove(path.Join(s.dir, bucket, ".meta_"+key)) + + return os.Remove(location) +} + +// Get implements S3. +func (s *s3_dir) Get(ctx context.Context, bucket string, key string) (*Object, error) { + location := path.Join(s.dir, bucket, key) + info, err := os.Stat(location) + if err != nil { + return nil, err + } + + if info.IsDir() { + return nil, fmt.Errorf("target is dir") + } + + var ( + f io.ReadCloser + bs []byte + obj = &Object{} + ) + + if f, err = os.Open(location); err != nil { + return nil, err + } + + obj.Body = f + + if bs, err = os.ReadFile(path.Join(s.dir, bucket, ".meta_"+key)); err != nil { + log.Warn(ctx, "s3.dir: open file meta err, err = %v", err) + return obj, nil + } + + m := new(meta) + if err = json.Unmarshal(bs, m); err != nil { + log.Warn(ctx, "s3.dir: unmarshal meta err, err = %v", err) + return obj, nil + } + + obj.ContentType = m.ContentType + obj.Size = m.Size + obj.ExpireAt = m.ExpireAt + + return obj, nil +} + +// Put implements S3. +func (s *s3_dir) Put(ctx context.Context, bucket string, key string, obj *Object) error { + if bucket != "" { + os.MkdirAll(path.Join(s.dir, bucket), 0o755) + } + + location := path.Join(s.dir, bucket, key) + ml := path.Join(s.dir, bucket, ".meta_"+key) + + bs, err := io.ReadAll(obj.Body) + if err != nil { + return err + } + + if obj.Size != 0 && obj.Size != int64(len(bs)) { + return fmt.Errorf("object size mismatch") + } + + obj.Size = int64(len(bs)) + + if obj.ContentType == "" { + obj.ContentType = http.DetectContentType(lo.If(len(bs) >= 128, bs[:128]).Else(bs)) + } + + if err = os.WriteFile(location, bs, 0o644); err != nil { + return err + } + + m := &meta{ + ContentType: obj.ContentType, + Size: obj.Size, + ExpireAt: obj.ExpireAt, + } + + ms, _ := json.Marshal(m) + os.WriteFile(ml, ms, 0o644) + + return nil +} + +func newDirClient(ctx context.Context, dir string) (S3, error) { + dir = filepath.ToSlash(dir) + info, err := os.Stat(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + if err = os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + } + + return nil, err + } + + if !info.IsDir() { + return nil, fmt.Errorf("target dir exist but not dir") + } + + c := &s3_dir{ctx: ctx, dir: dir} + + // todo: expire files + go func() { + ticker := time.NewTicker(10 * time.Minute) + for { + select { + case <-ctx.Done(): + return + case t := <-ticker.C: + log.Debug(ctx, "s3.by_dir: start clean up s3 dir @%s", t.Format("2006-01-02T15:04:05")) + c.clean() + } + } + }() + + return c, nil +} diff --git a/internal/database/s3/by_s3.go b/internal/database/s3/by_s3.go new file mode 100644 index 0000000..c4e90bd --- /dev/null +++ b/internal/database/s3/by_s3.go @@ -0,0 +1,112 @@ +package s3 + +import ( + "context" + "time" + + "ultone/internal/log" + "ultone/internal/tool" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +type s3_client struct { + client *s3.Client +} + +// Delete implements S3. +func (s *s3_client) Delete(ctx context.Context, bucket string, key string) error { + var err error + + if _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }); err != nil { + return err + } + + return nil +} + +// Get implements S3. +func (s *s3_client) Get(ctx context.Context, bucket string, key string) (*Object, error) { + var ( + err error + res *s3.GetObjectOutput + ) + + if res, err = s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }); err != nil { + return nil, err + } + + return &Object{ + ContentType: *res.ContentType, + Body: res.Body, + ExpireAt: res.Expires.UnixMilli(), + Size: *res.ContentLength, + }, nil +} + +// Put implements S3. +func (s *s3_client) Put(ctx context.Context, bucket string, key string, obj *Object) error { + var err error + + if _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + ACL: types.ObjectCannedACLPublicRead, + Body: obj.Body, + ContentType: aws.String(obj.ContentType), + Expires: aws.Time(time.UnixMilli(obj.ExpireAt)), + }); err != nil { + return err + } + + return nil +} + +func newS3Client(ctx context.Context, endpoint, access, key string) (S3, error) { + var ( + err error + sdkConfig aws.Config + output *s3.ListBucketsOutput + ) + + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + }, nil + }) + + if sdkConfig, err = config.LoadDefaultConfig( + ctx, + config.WithEndpointResolverWithOptions(customResolver), + ); err != nil { + return nil, err + } + + s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) { + // o.BaseEndpoint = aws.String(endpoint) + // o.EndpointResolverV2 = &resolverV2{} + o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(access, key, "")) + o.UsePathStyle = true + o.Region = "auto" + }) + + if output, err = s3Client.ListBuckets(tool.Timeout(5), &s3.ListBucketsInput{}); err != nil { + return nil, err + } + + for _, item := range output.Buckets { + log.Debug(ctx, "s3.New: list bucket name = %s", *item.Name) + } + + return &s3_client{client: s3Client}, nil +} diff --git a/internal/database/s3/new.go b/internal/database/s3/new.go new file mode 100644 index 0000000..50e3c2b --- /dev/null +++ b/internal/database/s3/new.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "context" + "fmt" + "net/url" +) + +var Default S3 + +func New(ctx context.Context, uri string) (S3, error) { + ins, err := url.Parse(uri) + if err != nil { + return nil, err + } + + switch ins.Scheme { + case "http", "https": + if ins.User == nil { + return nil, fmt.Errorf("missing access or key") + } + access := ins.User.Username() + key, _ := ins.User.Password() + return newS3Client(ctx, fmt.Sprintf("%s://%s", ins.Scheme, ins.Host), access, key) + case "dir": + return newDirClient(ctx, ins.Host) + default: + return nil, fmt.Errorf("invalid new s3 uri scheme: %s", ins.Scheme) + } +} + +func Init(ctx context.Context, uri string) (err error) { + Default, err = New(ctx, uri) + return err +} diff --git a/internal/database/s3/s3.go b/internal/database/s3/s3.go new file mode 100644 index 0000000..e7bab45 --- /dev/null +++ b/internal/database/s3/s3.go @@ -0,0 +1,25 @@ +package s3 + +import ( + "context" + "io" +) + +type meta struct { + ContentType string `json:"content_type"` + Size int64 `json:"size"` + ExpireAt int64 `json:"expire_at"` +} + +type Object struct { + ContentType string + Body io.ReadCloser + Size int64 + ExpireAt int64 +} + +type S3 interface { + Get(ctx context.Context, bucket, key string) (*Object, error) + Put(ctx context.Context, bucket, key string, obj *Object) error + Delete(ctx context.Context, bucket, key string) error +} diff --git a/internal/tool/human.go b/internal/tool/human.go index 2c7ce71..92144e3 100644 --- a/internal/tool/human.go +++ b/internal/tool/human.go @@ -22,3 +22,29 @@ func HumanDuration(nano int64) string { return fmt.Sprintf("%6.2f%s", duration, unit) } + +func HumanSize(size int64) string { + const ( + _ = iota + KB = 1 << (10 * iota) // 1 KB = 1024 bytes + MB // 1 MB = 1024 KB + GB // 1 GB = 1024 MB + TB // 1 TB = 1024 GB + PB // 1 PB = 1024 TB + ) + + switch { + case size >= PB: + return fmt.Sprintf("%.2f PB", float64(size)/PB) + case size >= TB: + return fmt.Sprintf("%.2f TB", float64(size)/TB) + case size >= GB: + return fmt.Sprintf("%.2f GB", float64(size)/GB) + case size >= MB: + return fmt.Sprintf("%.2f MB", float64(size)/MB) + case size >= KB: + return fmt.Sprintf("%.2f KB", float64(size)/KB) + default: + return fmt.Sprintf("%d bytes", size) + } +}