From 0bcb138fd5586bd3020e6733a5df48741a0cfb3b Mon Sep 17 00:00:00 2001 From: zhaoyupeng Date: Mon, 29 Dec 2025 23:01:42 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=BA=86=20imager=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=20package=20refactor:=20=E5=B0=86=20=20image?= =?UTF-8?q?s=20=E7=9A=84=E8=8E=B7=E5=8F=96=E5=88=86=E6=95=A3=E5=88=B0?= =?UTF-8?q?=E5=90=84=E4=B8=AA=E7=BB=84=E4=BB=B6=E9=87=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/cmd/make.go | 2 + internal/cmd/makecmd/all.go | 130 ++++ internal/cmd/makecmd/app.go | 2 +- internal/cmd/makecmd/redis.go | 8 +- internal/cmd/makecmd/registry.go | 28 + .../controller/installer/installer.redis.go | 5 + internal/controller/maker/app.oem.go | 2 +- internal/controller/maker/flannel.go | 22 + internal/controller/maker/hsnet.go | 4 +- internal/controller/maker/image.go | 15 - internal/controller/maker/longhorn.go | 43 +- internal/controller/maker/registry.go | 73 ++ internal/controller/maker/yosguard.go | 41 +- internal/opt/opt.go | 4 + pkg/imager/pull.go | 648 ++++++++++++++++++ pkg/imager/pull_test.go | 429 ++++++++++++ pkg/resource/nginx/client.conf | 10 +- pkg/resource/nginx/web.conf | 4 + pkg/resource/resource.go | 3 + pkg/resource/yaml/registry.yaml | 102 +++ 20 files changed, 1519 insertions(+), 56 deletions(-) create mode 100644 internal/cmd/makecmd/all.go create mode 100644 internal/cmd/makecmd/registry.go create mode 100644 internal/controller/installer/installer.redis.go create mode 100644 internal/controller/maker/registry.go create mode 100644 pkg/imager/pull.go create mode 100644 pkg/imager/pull_test.go create mode 100644 pkg/resource/yaml/registry.yaml diff --git a/internal/cmd/make.go b/internal/cmd/make.go index 1f5ba23..698a176 100644 --- a/internal/cmd/make.go +++ b/internal/cmd/make.go @@ -50,6 +50,7 @@ func makeCmd() *cobra.Command { _cmd.PersistentFlags().StringVar(&opt.Cfg.Make.Dir, "dir", "/root/hsv2-installation", "make base directory") _cmd.AddCommand( + makecmd.ALL(), makecmd.Images(), makecmd.Binaries(), makecmd.Flannel(), @@ -60,6 +61,7 @@ func makeCmd() *cobra.Command { makecmd.EMQX(), makecmd.Minio(), makecmd.Yosguard(), + makecmd.Registry(), makecmd.LessDNS(), makecmd.HSNet(), makecmd.ConfigMap(), diff --git a/internal/cmd/makecmd/all.go b/internal/cmd/makecmd/all.go new file mode 100644 index 0000000..3619351 --- /dev/null +++ b/internal/cmd/makecmd/all.go @@ -0,0 +1,130 @@ +package makecmd + +import ( + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/controller/maker" + "yizhisec.com/hsv2/forge/internal/opt" +) + +func ALL() *cobra.Command { + var ( + _workdir string + ) + + _cmd := &cobra.Command{ + Use: "all", + Short: "Make all", + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + mk = maker.NewMaker(_workdir) + ) + + if err = mk.Images(cmd.Context()); err != nil { + return err + } + + if err = mk.Binary(cmd.Context()); err != nil { + return err + } + + if err = mk.Registry(cmd.Context(), "50Gi"); err != nil { + return err + } + + if err = mk.Flannel(cmd.Context(), "host-gw"); err != nil { + return err + } + + if err = mk.Longhorn(cmd.Context(), 2); err != nil { + return err + } + + if err = mk.MySQL(cmd.Context()); err != nil { + return err + } + + if err = mk.Redis(cmd.Context()); err != nil { + return err + } + + if err = mk.Elastic(cmd.Context()); err != nil { + return err + } + + if err = mk.EMQX(cmd.Context()); err != nil { + return err + } + + if err = mk.Minio(cmd.Context(), "100Gi"); err != nil { + return err + } + + if err = mk.Yosguard(cmd.Context()); err != nil { + return err + } + + if err = mk.LessDNS(cmd.Context()); err != nil { + return err + } + + if err = mk.HSNet(cmd.Context()); err != nil { + return err + } + + if err = mk.ConfigMap(cmd.Context()); err != nil { + return err + } + + if err = mk.Proxy(cmd.Context()); err != nil { + return err + } + + if err = mk.Seafile(cmd.Context()); err != nil { + return err + } + + if err = mk.Proxy(cmd.Context()); err != nil { + return err + } + + if err = mk.Seafile(cmd.Context()); err != nil { + return err + } + + if err = mk.AppOEM(cmd.Context(), "standard", 2); err != nil { + return err + } + + if err = mk.AppUser(cmd.Context(), 2); err != nil { + return err + } + + if err = mk.AppClient(cmd.Context(), 2); err != nil { + return err + } + + if err = mk.AppGateway(cmd.Context(), 2); err != nil { + return err + } + + if err = mk.AppFront(cmd.Context(), "standard", 2); err != nil { + return err + } + + if err = mk.AppMie(cmd.Context(), 2); err != nil { + return err + } + + if err = mk.AppNginx(cmd.Context()); err != nil { + return err + } + + return nil + }, + } + + _cmd.PersistentFlags().StringVar(&_workdir, "workdir", opt.DefaultWorkdir, "Work directory") + + return _cmd +} diff --git a/internal/cmd/makecmd/app.go b/internal/cmd/makecmd/app.go index aae755a..daaa414 100644 --- a/internal/cmd/makecmd/app.go +++ b/internal/cmd/makecmd/app.go @@ -112,7 +112,7 @@ func appOEM() *cobra.Command { Short: "Make OEM App", RunE: func(cmd *cobra.Command, args []string) error { mk := maker.NewMaker(opt.Cfg.Make.Dir) - return mk.AppOEM(cmd.Context(), replica, vendor) + return mk.AppOEM(cmd.Context(), vendor, replica) }, } diff --git a/internal/cmd/makecmd/redis.go b/internal/cmd/makecmd/redis.go index aeef4c9..d5fe6c2 100644 --- a/internal/cmd/makecmd/redis.go +++ b/internal/cmd/makecmd/redis.go @@ -1,6 +1,8 @@ package makecmd import ( + "fmt" + "github.com/spf13/cobra" "yizhisec.com/hsv2/forge/internal/controller/maker" "yizhisec.com/hsv2/forge/internal/opt" @@ -10,7 +12,7 @@ func Redis() *cobra.Command { var ( replicas int password string - storage string + storage int ) _cmd := &cobra.Command{ @@ -23,14 +25,14 @@ func Redis() *cobra.Command { cmd.Context(), maker.WithRedisReplicaCount(replicas), maker.WithRedisPassword(password), - maker.WithRedisStorage(storage), + maker.WithRedisStorage(fmt.Sprintf("%dGi")), ) }, } _cmd.Flags().IntVar(&replicas, "replica-count", 2, "Redis 副本数") _cmd.Flags().StringVar(&password, "password", "", "Redis 密码") - _cmd.Flags().StringVar(&storage, "storage-size", "5Gi", "Redis 存储大小(如: 5Gi)") + _cmd.Flags().IntVar(&storage, "storage-size", 5, "Redis 存储大小(单位Gi)如: 5") return _cmd } diff --git a/internal/cmd/makecmd/registry.go b/internal/cmd/makecmd/registry.go new file mode 100644 index 0000000..040adcc --- /dev/null +++ b/internal/cmd/makecmd/registry.go @@ -0,0 +1,28 @@ +package makecmd + +import ( + "fmt" + + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/controller/maker" + "yizhisec.com/hsv2/forge/internal/opt" +) + +func Registry() *cobra.Command { + var ( + size int + ) + + _cmd := &cobra.Command{ + Use: "registry", + Short: "Make registry dependency", + RunE: func(cmd *cobra.Command, args []string) error { + mk := maker.NewMaker(opt.Cfg.Make.Dir) + return mk.Registry(cmd.Context(), fmt.Sprintf("%dGi", size)) + }, + } + + _cmd.Flags().IntVar(&size, "storage-size", 50, "Redis 存储大小(单位Gi)如: 100") + + return _cmd +} diff --git a/internal/controller/installer/installer.redis.go b/internal/controller/installer/installer.redis.go new file mode 100644 index 0000000..ff2799c --- /dev/null +++ b/internal/controller/installer/installer.redis.go @@ -0,0 +1,5 @@ +package installer + +func (i *installer) Redis() error { + return nil +} diff --git a/internal/controller/maker/app.oem.go b/internal/controller/maker/app.oem.go index 7e15810..322ff2f 100644 --- a/internal/controller/maker/app.oem.go +++ b/internal/controller/maker/app.oem.go @@ -13,7 +13,7 @@ import ( "yizhisec.com/hsv2/forge/pkg/resource" ) -func (m *maker) AppOEM(ctx context.Context, replica int, vendor string) error { +func (m *maker) AppOEM(ctx context.Context, vendor string, replica int) error { const ( _nginx = `user root; worker_processes auto; diff --git a/internal/controller/maker/flannel.go b/internal/controller/maker/flannel.go index 7621ecf..0f74eb7 100644 --- a/internal/controller/maker/flannel.go +++ b/internal/controller/maker/flannel.go @@ -7,6 +7,7 @@ import ( "path/filepath" "gitea.loveuer.com/yizhisec/pkg3/logger" + "yizhisec.com/hsv2/forge/pkg/model" "yizhisec.com/hsv2/forge/pkg/resource" ) @@ -33,6 +34,27 @@ func (m *maker) Flannel(ctx context.Context, mode string) error { return err } + var images = []*model.Image{ + {Name: "ghcr.io/flannel-io/flannel:v0.27.4", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel:v0.27.4", Save: "flannel.tar"}, + {Name: "ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Save: "flannel-cni-plugin.tar"}, + } + + logger.Debug("MakeFlannel: 开始获取镜像...") + for _, image := range images { + opts := []ImageOpt{ + WithImageFallback(image.Fallback), + WithImageSave(filepath.Join(location, image.Save)), + WithImageForcePull(image.Force), + } + + if err := m.Image(ctx, image.Name, opts...); err != nil { + logger.Error("❌ MakeFlannel: 获取镜像失败: %s, 可以手动获取后重试", image.Name) + logger.Debug("❌ MakeFlannel: 获取镜像失败: %s, %v", image.Name, err) + return err + } + } + logger.Debug("MakeFlannel: 获取镜像成功!!!") + logger.Info("✅ 构建 flannel 成功!!!") return nil diff --git a/internal/controller/maker/hsnet.go b/internal/controller/maker/hsnet.go index c3b3e56..eb215b4 100644 --- a/internal/controller/maker/hsnet.go +++ b/internal/controller/maker/hsnet.go @@ -46,10 +46,8 @@ ExecStart=/usr/local/bin/k0s ctr -n hs-net run \ # --cgroup host \ # --env RUSTFLAGS="-C target-cpu=nehalem" \ # 重启策略 -Restart=on-failure +Restart=always RestartSec=5s -StartLimitInterval=60s -StartLimitBurst=5 # 资源限制（按需调整） MemoryLimit=2G diff --git a/internal/controller/maker/image.go b/internal/controller/maker/image.go index 7d0bce4..56c284c 100644 --- a/internal/controller/maker/image.go +++ b/internal/controller/maker/image.go @@ -119,21 +119,6 @@ func (m *maker) Images(ctx context.Context) error { {Name: "quay.io/k0sproject/metrics-server:v0.7.2-0", Fallback: "", Save: "k0s.metrics-server.tar"}, {Name: "quay.io/k0sproject/pause:3.10.1", Fallback: "", Save: "k0s.pause.tar"}, - {Name: "ghcr.io/flannel-io/flannel:v0.27.4", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel:v0.27.4", Save: "flannel.tar"}, - {Name: "ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Save: "flannel-cni-plugin.tar"}, - - {Name: "docker.io/longhornio/longhorn-engine:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-engine:v1.10.0", Save: "longhorn.longhorn-engine.tar"}, - {Name: "docker.io/longhornio/longhorn-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-manager:v1.10.0", Save: "longhorn.longhorn-manager.tar"}, - {Name: "docker.io/longhornio/longhorn-instance-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-instance-manager:v1.10.0", Save: "longhorn.longhorn-instance-manager.tar"}, - {Name: "docker.io/longhornio/longhorn-share-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-share-manager:v1.10.0", Save: "longhorn.longhorn-share-manager.tar"}, - {Name: "docker.io/longhornio/longhorn-ui:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-ui:v1.10.0", Save: "longhorn.longhorn-ui.tar"}, - {Name: "docker.io/longhornio/csi-snapshotter:v8.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-snapshotter:v8.3.0-20250826", Save: "longhorn.csi-snapshotter.tar"}, - {Name: "docker.io/longhornio/csi-resizer:v1.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-resizer:v1.14.0-20250826", Save: "longhorn.csi-resizer.tar"}, - {Name: "docker.io/longhornio/csi-provisioner:v5.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-provisioner:v5.3.0-20250826", Save: "longhorn.csi-provisioner.tar"}, - {Name: "docker.io/longhornio/livenessprobe:v2.16.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/livenessprobe:v2.16.0-20250826", Save: "longhorn.livenessprobe.tar"}, - {Name: "docker.io/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Save: "longhorn.csi-node-driver-registrar.tar"}, - {Name: "docker.io/longhornio/csi-attacher:v4.9.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-attacher:v4.9.0-20250826", Save: "longhorn.csi-attacher.tar"}, - {Name: "hub.yizhisec.com/external/alpine:3.22.2", Fallback: "", Save: "alpine.tar", Force: true}, {Name: "hub.yizhisec.com/external/nginx:1.29.1-alpine3.22", Fallback: "", Save: "nginx.1.29.1-alpine3.22.tar"}, diff --git a/internal/controller/maker/longhorn.go b/internal/controller/maker/longhorn.go index 026bce7..b89b8c9 100644 --- a/internal/controller/maker/longhorn.go +++ b/internal/controller/maker/longhorn.go @@ -8,6 +8,7 @@ import ( "gitea.loveuer.com/yizhisec/pkg3/logger" "yizhisec.com/hsv2/forge/pkg/downloader" + "yizhisec.com/hsv2/forge/pkg/model" ) func (m *maker) Longhorn(ctx context.Context, replica int) error { @@ -23,16 +24,16 @@ persistence: ) var ( - err error - chartURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/charts/longhorn-1.10.0.tgz" - longhornDir = filepath.Join(m.workdir, "dependency", "longhorn") - chartFile = filepath.Join(longhornDir, "longhorn-1.10.0.tgz") - valuesFile = filepath.Join(longhornDir, "values.yaml") + err error + chartURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/charts/longhorn-1.10.0.tgz" + location = filepath.Join(m.workdir, "dependency", "longhorn") + chartFile = filepath.Join(location, "longhorn-1.10.0.tgz") + valuesFile = filepath.Join(location, "values.yaml") ) logger.Info("☑️ 开始准备 Longhorn 资源...") logger.Debug("下载地址: %s", chartURL) - logger.Debug("目标目录: %s", longhornDir) + logger.Debug("目标目录: %s", location) if err = os.MkdirAll(filepath.Join(m.workdir, "dependency", "longhorn"), 0755); err != nil { return err @@ -57,6 +58,36 @@ persistence: return err } + var images = []*model.Image{ + {Name: "docker.io/longhornio/longhorn-engine:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-engine:v1.10.0", Save: "longhorn.longhorn-engine.tar"}, + {Name: "docker.io/longhornio/longhorn-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-manager:v1.10.0", Save: "longhorn.longhorn-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-instance-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-instance-manager:v1.10.0", Save: "longhorn.longhorn-instance-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-share-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-share-manager:v1.10.0", Save: "longhorn.longhorn-share-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-ui:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-ui:v1.10.0", Save: "longhorn.longhorn-ui.tar"}, + {Name: "docker.io/longhornio/csi-snapshotter:v8.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-snapshotter:v8.3.0-20250826", Save: "longhorn.csi-snapshotter.tar"}, + {Name: "docker.io/longhornio/csi-resizer:v1.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-resizer:v1.14.0-20250826", Save: "longhorn.csi-resizer.tar"}, + {Name: "docker.io/longhornio/csi-provisioner:v5.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-provisioner:v5.3.0-20250826", Save: "longhorn.csi-provisioner.tar"}, + {Name: "docker.io/longhornio/livenessprobe:v2.16.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/livenessprobe:v2.16.0-20250826", Save: "longhorn.livenessprobe.tar"}, + {Name: "docker.io/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Save: "longhorn.csi-node-driver-registrar.tar"}, + {Name: "docker.io/longhornio/csi-attacher:v4.9.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-attacher:v4.9.0-20250826", Save: "longhorn.csi-attacher.tar"}, + } + + logger.Debug("MakeLonghorn: 开始获取镜像...") + for _, image := range images { + opts := []ImageOpt{ + WithImageFallback(image.Fallback), + WithImageSave(filepath.Join(location, image.Save)), + WithImageForcePull(image.Force), + } + + if err := m.Image(ctx, image.Name, opts...); err != nil { + logger.Error("❌ MakeLonghorn: 获取镜像失败: %s, 可以手动获取后重试", image.Name) + logger.Debug("❌ MakeLonghorn: 获取镜像失败: %s, %v", image.Name, err) + return err + } + } + logger.Debug("MakeLonghorn: 获取镜像成功!!!") + logger.Info("✅ 成功创建 longhorn 资源!!!") return nil diff --git a/internal/controller/maker/registry.go b/internal/controller/maker/registry.go new file mode 100644 index 0000000..4a03071 --- /dev/null +++ b/internal/controller/maker/registry.go @@ -0,0 +1,73 @@ +package maker + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func (m *maker) Registry(ctx context.Context, storage string) error { + const ( + _registryToml = ` +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "" + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.96.123.45:80"] + endpoint = ["http://10.96.123.45:80"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.96.123.45:80".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.96.123.45:80".auth]` + ) + + var ( + err error + location = filepath.Join(m.workdir, "dependency", "registry") + imgName = "docker.io/library/registry:2.8.3" + uiImgName = "docker.io/quiq/registry-ui:0.11.0" + imgFallback = "docker-mirror.yizhisec.com/library/registry:2.8.3" + uiImgFallback = "docker-mirror.yizhisec.com/quiq/registry-ui:0.11.0" + ) + + logger.Info("☑️ 开始创建依赖: registry...") + + logger.Debug("☑️ 创建 dir: %s ...", location) + if err = os.MkdirAll(location, 0755); err != nil { + logger.Debug("❌ 创建 dir %s 失败: %v", location, err) + return err + } + + yamlFile := filepath.Join(location, "registry.yaml") + content := fmt.Sprintf(resource.YAMLRegistry, storage) + logger.Debug("写入 yaml 文件: %s ...", yamlFile) + if os.WriteFile(yamlFile, []byte(content), 0644); err != nil { + logger.Debug("❌ 写入 yaml 文件 %s 失败: %v", yamlFile, err) + return err + } + + logger.Debug("开始准备镜像...") + imgFile := filepath.Join(location, "registry.tar") + if err = m.Image(ctx, imgName, WithImageFallback(imgFallback), WithImageSave(imgFile)); err != nil { + logger.Debug("❌ 准备镜像 %s 失败: %v", imgName, err) + return err + } + imgFile = filepath.Join(location, "registry-ui.tar") + if err = m.Image(ctx, uiImgName, WithImageFallback(uiImgFallback), WithImageSave(imgFile)); err != nil { + logger.Debug("❌ 准备镜像 %s 失败: %v", uiImgName, err) + return err + } + + logger.Debug("写入 registry.toml 文件: %s ...", filepath.Join(location, "registry.toml")) + if os.WriteFile(filepath.Join(location, "registry.toml"), []byte(_registryToml), 0644); err != nil { + logger.Debug("❌ 写入 registry.toml 文件 %s 失败: %v", filepath.Join(location, "registry.toml"), err) + return err + } + + logger.Info("✅ 创建依赖成功: registry!!!") + + return nil +} diff --git a/internal/controller/maker/yosguard.go b/internal/controller/maker/yosguard.go index 90bd24d..203e3df 100644 --- a/internal/controller/maker/yosguard.go +++ b/internal/controller/maker/yosguard.go @@ -16,32 +16,21 @@ type yosguardOpt struct{} func (m *maker) Yosguard(ctx context.Context, opts ...YosguardOpt) error { const ( - configTemplate = ` -Web: - # default listen in docker0 - Host: 172.17.0.1 - Port: 7788 - -UUIDFilePath: /etc/yosguard/uuid - -# 心跳间隔: 单位秒,默认为5 -HeartbeatDuration: 5 - -# 控制器 yosguard 地址 + _config = ` +AsController: true +AsGateway: false ControllerServer: Host: dasheng.zhsftech.debug - Port: 443 - -# True: 作为控制器运行; False: 不作为控制器运行 -AsController: true - -# True: 作为网关运行; False: 不作为网关运行 -AsGateway: false - + Port: 443 Database: SQLite: - DBPath: "/etc/yosguard/db/yosguard.db" - SQLPath: "/etc/yosguard/db/create.sql"` + DBPath: /etc/yosguard/db/yosguard.db +HeartbeatDuration: 5 +UUIDFilePath: /etc/yosguard/uuid +Web: + Host: __ip__ + Port: 7788 +` systemdService = ` [Unit] @@ -86,12 +75,12 @@ WantedBy=multi-user.target` } logger.Debug("✅ maker.Yosguard: 下载 yosguard 成功, url = %s", binURL) - logger.Debug("☑️ maker.Yosguard: 写入 config_template.yml 文件..., dest = %s", filepath.Join(location, "config_template.yml")) - if err := os.WriteFile(filepath.Join(location, "config_template.yml"), []byte(configTemplate), 0644); err != nil { - logger.Debug("❌ maker.Yosguard: 写入 config_template.yml 失败, dest = %s, err = %v", filepath.Join(location, "config_template.yml"), err) + logger.Debug("☑️ maker.Yosguard: 写入 config.yml 文件..., dest = %s", filepath.Join(location, "config.yml")) + if err := os.WriteFile(filepath.Join(location, "config.yml"), []byte(_config), 0644); err != nil { + logger.Debug("❌ maker.Yosguard: 写入 config.yml 失败, dest = %s, err = %v", filepath.Join(location, "config.yml"), err) return err } - logger.Debug("✅ maker.Yosguard: 写入 config_template.yml 文件成功, dest = %s", filepath.Join(location, "config_template.yml")) + logger.Debug("✅ maker.Yosguard: 写入 config.yml 文件成功, dest = %s", filepath.Join(location, "config.yml")) logger.Debug("☑️ maker.Yosguard: 写入 create.sql 文件..., dest = %s", filepath.Join(location, "create.sql")) if err := os.WriteFile(filepath.Join(location, "create.sql"), resource.SQLYosguard, 0644); err != nil { diff --git a/internal/opt/opt.go b/internal/opt/opt.go index 8ade8c3..67b0e65 100644 --- a/internal/opt/opt.go +++ b/internal/opt/opt.go @@ -14,6 +14,10 @@ var ( Cfg = &config{} ) +const ( + DefaultWorkdir = "/root/hsv2-installation" +) + var ( StorageSizeReg = regexp.MustCompile(`^\d+(\.\d+)?[EPTGMK]i?$`) EmailReg = regexp.MustCompile(`^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$`) diff --git a/pkg/imager/pull.go b/pkg/imager/pull.go new file mode 100644 index 0000000..50ef2bc --- /dev/null +++ b/pkg/imager/pull.go @@ -0,0 +1,648 @@ +package imager + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +// PullOption is a functional option for configuring image pull +type PullOption func(*pullOption) + +type pullOption struct { + Proxy string // http or socks5 proxy + Rename string // pull image and rename it + PlainHTTP bool // use http instead of https + SkipTLSVerify bool // skip TLS certificate verification + Username string // registry username for authentication + Password string // registry password for authentication + RetryTimes int // retry times for sync/proxy registries, default is 3 + RetryDelay time.Duration // delay between retries, default is 2 seconds +} + +// WithProxy sets the proxy for pulling images +func WithProxy(proxy string) PullOption { + return func(o *pullOption) { + o.Proxy = proxy + } +} + +// WithRename sets a new name for the pulled image +func WithRename(name string) PullOption { + return func(o *pullOption) { + o.Rename = name + } +} + +// WithPlainHTTP enables plain HTTP (no TLS) +func WithPlainHTTP() PullOption { + return func(o *pullOption) { + o.PlainHTTP = true + } +} + +// WithSkipTLSVerify skips TLS certificate verification +func WithSkipTLSVerify() PullOption { + return func(o *pullOption) { + o.SkipTLSVerify = true + } +} + +// WithAuth sets authentication credentials +func WithAuth(username, password string) PullOption { + return func(o *pullOption) { + o.Username = username + o.Password = password + } +} + +// WithRetry sets the retry times for pulling images +func WithRetry(times int) PullOption { + return func(o *pullOption) { + if times < 0 { + times = 0 + } + o.RetryTimes = times + } +} + +// WithRetryDelay sets the delay between retries +func WithRetryDelay(delay time.Duration) PullOption { + return func(o *pullOption) { + o.RetryDelay = delay + } +} + +// manifestV2 represents Docker manifest v2 schema +type manifestV2 struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config struct { + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Digest string `json:"digest"` + } `json:"config"` + Layers []struct { + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Digest string `json:"digest"` + } `json:"layers"` +} + +// imageReference parses image reference into registry, repository, and tag +type imageReference struct { + Registry string + Repository string + Tag string +} + +// parseImageReference parses an image name into its components +func parseImageReference(name string) (*imageReference, error) { + ref := &imageReference{ + Tag: "latest", + } + + // First, split by / to separate registry from repository + // This handles cases like localhost:5000/myimage:tag + slashParts := strings.SplitN(name, "/", 2) + + var registryPart, repoPart string + + if len(slashParts) == 1 { + // No slash, it's just image:tag (e.g., alpine:3.19) + repoPart = slashParts[0] + ref.Registry = "registry-1.docker.io" + } else { + // Has slash, check if first part is a registry + // Registry contains . or : (for host:port) + if strings.Contains(slashParts[0], ".") || strings.Contains(slashParts[0], ":") { + // First part is a registry + registryPart = slashParts[0] + repoPart = slashParts[1] + ref.Registry = registryPart + } else { + // First part is namespace (e.g., library/alpine) + repoPart = name + ref.Registry = "registry-1.docker.io" + } + } + + // Now parse the repository and tag from repoPart + // Split on last : to get tag (in case repository name contains /) + lastColon := strings.LastIndex(repoPart, ":") + if lastColon > 0 { + ref.Repository = repoPart[:lastColon] + ref.Tag = repoPart[lastColon+1:] + } else { + ref.Repository = repoPart + } + + // For docker.io, add library/ prefix if no namespace + if ref.Registry == "registry-1.docker.io" && !strings.Contains(ref.Repository, "/") { + ref.Repository = "library/" + ref.Repository + } + + return ref, nil +} + +// PullImage pulls an OCI image using HTTP requests and saves it as a tar archive +func PullImage(ctx context.Context, name string, store io.Writer, opts ...PullOption) error { + options := &pullOption{ + RetryTimes: 3, // default retry times + RetryDelay: 2 * time.Second, // default retry delay + } + for _, opt := range opts { + opt(options) + } + + logger.Debug("Pulling image: %s with options: %+v", name, options) + + // Parse image reference + ref, err := parseImageReference(name) + if err != nil { + return fmt.Errorf("failed to parse image reference: %w", err) + } + + logger.Debug("Parsed image reference: registry=%s, repository=%s, tag=%s", ref.Registry, ref.Repository, ref.Tag) + + // Create HTTP client + client := createHTTPClient(options) + + // Retry logic for sync/proxy registries + var lastErr error + for attempt := 0; attempt <= options.RetryTimes; attempt++ { + if attempt > 0 { + logger.Debug("Retry attempt %d/%d for image: %s", attempt, options.RetryTimes, name) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(options.RetryDelay): + // Continue after delay + } + } + + // Get auth token if needed + token, err := getAuthToken(ctx, client, ref, options) + if err != nil { + lastErr = fmt.Errorf("failed to get auth token: %w", err) + logger.Debug("Attempt %d failed: %v", attempt+1, lastErr) + continue + } + + // Get manifest + manifest, err := getManifest(ctx, client, ref, token, options) + if err != nil { + lastErr = fmt.Errorf("failed to get manifest: %w", err) + logger.Debug("Attempt %d failed: %v", attempt+1, lastErr) + continue + } + + logger.Debug("Got manifest with %d layers", len(manifest.Layers)) + + // Download and create tar archive + if err := createImageTar(ctx, client, ref, manifest, token, store, name, options); err != nil { + lastErr = fmt.Errorf("failed to create image tar: %w", err) + logger.Debug("Attempt %d failed: %v", attempt+1, lastErr) + continue + } + + // Success! + logger.Info("Successfully pulled image: %s", name) + return nil + } + + // All retries exhausted + if options.RetryTimes > 0 { + return fmt.Errorf("failed to pull image after %d retries: %w", options.RetryTimes+1, lastErr) + } + return lastErr +} + +// createHTTPClient creates an HTTP client with the given options +func createHTTPClient(options *pullOption) *http.Client { + client := &http.Client{} + + // Configure TLS + if options.SkipTLSVerify { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client.Transport = transport + } + + // Configure proxy + if options.Proxy != "" { + proxyURL, err := url.Parse(options.Proxy) + if err == nil { + if client.Transport == nil { + client.Transport = &http.Transport{} + } + if transport, ok := client.Transport.(*http.Transport); ok { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + } + + return client +} + +// getAuthToken gets authentication token from registry +func getAuthToken(ctx context.Context, client *http.Client, ref *imageReference, options *pullOption) (string, error) { + // Try to access registry API v2 to get auth challenge + scheme := "https" + if options.PlainHTTP { + scheme = "http" + } + + apiURL := fmt.Sprintf("%s://%s/v2/", scheme, ref.Registry) + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return "", err + } + + // Add basic auth if provided + if options.Username != "" && options.Password != "" { + req.SetBasicAuth(options.Username, options.Password) + } + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + // If 200 OK, no auth needed + if resp.StatusCode == http.StatusOK { + return "", nil + } + + // Check for WWW-Authenticate header + authHeader := resp.Header.Get("WWW-Authenticate") + if authHeader == "" { + return "", nil + } + + // Parse auth challenge + token, err := fetchToken(ctx, client, authHeader, ref, options) + if err != nil { + return "", err + } + + return token, nil +} + +// fetchToken fetches authentication token from auth server +func fetchToken(ctx context.Context, client *http.Client, authHeader string, ref *imageReference, options *pullOption) (string, error) { + // Parse WWW-Authenticate header + // Format: Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/alpine:pull" + if !strings.HasPrefix(authHeader, "Bearer ") { + return "", nil + } + + params := make(map[string]string) + parts := strings.Split(authHeader[7:], ",") + for _, part := range parts { + part = strings.TrimSpace(part) + idx := strings.Index(part, "=") + if idx > 0 { + key := part[:idx] + value := strings.Trim(part[idx+1:], "\"") + params[key] = value + } + } + + realm, ok := params["realm"] + if !ok { + return "", fmt.Errorf("no realm in auth header") + } + + // Build token URL + tokenURL := realm + if service, ok := params["service"]; ok { + tokenURL += "?service=" + url.QueryEscape(service) + } + scope := fmt.Sprintf("repository:%s:pull", ref.Repository) + if params["scope"] != "" { + scope = params["scope"] + } + tokenURL += "&scope=" + url.QueryEscape(scope) + + req, err := http.NewRequestWithContext(ctx, "GET", tokenURL, nil) + if err != nil { + return "", err + } + + // Add basic auth if provided + if options.Username != "" && options.Password != "" { + req.SetBasicAuth(options.Username, options.Password) + } + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get token: %s", resp.Status) + } + + var tokenResp struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return "", err + } + + return tokenResp.Token, nil +} + +// getManifest fetches the image manifest +func getManifest(ctx context.Context, client *http.Client, ref *imageReference, token string, options *pullOption) (*manifestV2, error) { + scheme := "https" + if options.PlainHTTP { + scheme = "http" + } + + manifestURL := fmt.Sprintf("%s://%s/v2/%s/manifests/%s", scheme, ref.Registry, ref.Repository, ref.Tag) + req, err := http.NewRequestWithContext(ctx, "GET", manifestURL, nil) + if err != nil { + return nil, err + } + + // Set accept headers for manifest v2 (add both types) + req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json") + req.Header.Add("Accept", "application/vnd.oci.image.manifest.v1+json") + req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.list.v2+json") + + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + logger.Debug("Fetching manifest from: %s", manifestURL) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + logger.Debug("Manifest request failed: %s, body: %s", resp.Status, string(body)) + return nil, fmt.Errorf("failed to get manifest: %s", resp.Status) + } + + contentType := resp.Header.Get("Content-Type") + logger.Debug("Manifest Content-Type: %s", contentType) + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + // Check if it's a manifest list (multi-arch) + if strings.Contains(contentType, "manifest.list") || strings.Contains(contentType, "image.index") { + // Parse manifest list and select amd64/linux + var manifestList struct { + SchemaVersion int `json:"schemaVersion"` + Manifests []struct { + Digest string `json:"digest"` + Platform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + } `json:"platform"` + } `json:"manifests"` + } + if err := json.Unmarshal(body, &manifestList); err != nil { + return nil, fmt.Errorf("failed to parse manifest list: %w", err) + } + + logger.Debug("Found manifest list with %d manifests", len(manifestList.Manifests)) + + // Find amd64/linux manifest + var targetDigest string + for _, m := range manifestList.Manifests { + logger.Debug("Checking manifest: arch=%s, os=%s, digest=%s", + m.Platform.Architecture, m.Platform.OS, m.Digest) + if m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" { + targetDigest = m.Digest + break + } + } + if targetDigest == "" && len(manifestList.Manifests) > 0 { + // Fallback to first manifest with known platform + for _, m := range manifestList.Manifests { + if m.Platform.Architecture != "unknown" && m.Platform.OS != "unknown" { + targetDigest = m.Digest + logger.Debug("Using fallback manifest: arch=%s, os=%s", + m.Platform.Architecture, m.Platform.OS) + break + } + } + } + if targetDigest == "" { + return nil, fmt.Errorf("no suitable manifest found in manifest list") + } + + logger.Debug("Selected manifest digest: %s", targetDigest) + // Fetch the actual manifest by digest + return getManifestByDigest(ctx, client, ref, targetDigest, token, options) + } + + var manifest manifestV2 + if err := json.Unmarshal(body, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + logger.Debug("Parsed manifest: schemaVersion=%d, config.digest=%s, layers=%d", + manifest.SchemaVersion, manifest.Config.Digest, len(manifest.Layers)) + + return &manifest, nil +} + +// getManifestByDigest fetches a manifest by its digest +func getManifestByDigest(ctx context.Context, client *http.Client, ref *imageReference, digest, token string, options *pullOption) (*manifestV2, error) { + scheme := "https" + if options.PlainHTTP { + scheme = "http" + } + + manifestURL := fmt.Sprintf("%s://%s/v2/%s/manifests/%s", scheme, ref.Registry, ref.Repository, digest) + req, err := http.NewRequestWithContext(ctx, "GET", manifestURL, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json") + req.Header.Add("Accept", "application/vnd.oci.image.manifest.v1+json") + + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get manifest by digest: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var manifest manifestV2 + if err := json.Unmarshal(body, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// createImageTar downloads layers and creates a Docker-compatible tar archive +func createImageTar(ctx context.Context, client *http.Client, ref *imageReference, manifest *manifestV2, token string, store io.Writer, imageName string, options *pullOption) error { + // Create tar writer + tw := tar.NewWriter(store) + defer tw.Close() + + scheme := "https" + if options.PlainHTTP { + scheme = "http" + } + + // Download config blob + logger.Debug("Downloading config blob: %s", manifest.Config.Digest) + configBlob, err := downloadBlob(ctx, client, scheme, ref.Registry, ref.Repository, manifest.Config.Digest, token) + if err != nil { + return fmt.Errorf("failed to download config: %w", err) + } + + // Write config as json file + configFileName := strings.TrimPrefix(manifest.Config.Digest, "sha256:") + ".json" + if err := writeTarEntry(tw, configFileName, configBlob); err != nil { + return err + } + + // Download and write each layer + var layerFiles []string + for i, layer := range manifest.Layers { + logger.Debug("Downloading layer %d/%d: %s", i+1, len(manifest.Layers), layer.Digest) + layerBlob, err := downloadBlob(ctx, client, scheme, ref.Registry, ref.Repository, layer.Digest, token) + if err != nil { + return fmt.Errorf("failed to download layer %s: %w", layer.Digest, err) + } + + layerFileName := strings.TrimPrefix(layer.Digest, "sha256:") + "/layer.tar" + layerFiles = append(layerFiles, layerFileName) + + if err := writeTarEntry(tw, layerFileName, layerBlob); err != nil { + return err + } + } + + // Create manifest.json for Docker compatibility + if options.Rename != "" { + imageName = options.Rename + } + + manifestJSON := []map[string]interface{}{ + { + "Config": configFileName, + "RepoTags": []string{imageName}, + "Layers": layerFiles, + }, + } + + manifestData, err := json.Marshal(manifestJSON) + if err != nil { + return err + } + + if err := writeTarEntry(tw, "manifest.json", manifestData); err != nil { + return err + } + + // Create repositories file (legacy) + repositories := make(map[string]map[string]string) + repositories[imageName] = map[string]string{ref.Tag: strings.TrimPrefix(manifest.Config.Digest, "sha256:")[:12]} + reposData, err := json.Marshal(repositories) + if err != nil { + return err + } + + if err := writeTarEntry(tw, "repositories", reposData); err != nil { + return err + } + + return nil +} + +// downloadBlob downloads a blob from registry +func downloadBlob(ctx context.Context, client *http.Client, scheme, registry, repository, digest, token string) ([]byte, error) { + blobURL := fmt.Sprintf("%s://%s/v2/%s/blobs/%s", scheme, registry, repository, digest) + req, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil) + if err != nil { + return nil, err + } + + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download blob: %s", resp.Status) + } + + // Check if content is gzipped + var reader io.Reader = resp.Body + if resp.Header.Get("Content-Type") == "application/vnd.docker.image.rootfs.diff.tar.gzip" || + strings.Contains(resp.Header.Get("Content-Encoding"), "gzip") { + gzReader, err := gzip.NewReader(resp.Body) + if err == nil { + defer gzReader.Close() + reader = gzReader + } + } + + return io.ReadAll(reader) +} + +// writeTarEntry writes a file entry to tar archive +func writeTarEntry(tw *tar.Writer, name string, data []byte) error { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(data)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + if _, err := tw.Write(data); err != nil { + return err + } + + return nil +} diff --git a/pkg/imager/pull_test.go b/pkg/imager/pull_test.go new file mode 100644 index 0000000..d5bd31d --- /dev/null +++ b/pkg/imager/pull_test.go @@ -0,0 +1,429 @@ +package imager + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +// TestPullImage_PublicImage tests pulling a public image from Docker Hub +func TestPullImage_PublicImage(t *testing.T) { + logger.SetLogLevel(logger.LogLevelDebug) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Create temp directory for test output + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "alpine.tar") + outputFile = "./redis.alpine.tar" + + // Create output file + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Pull alpine image (small and commonly available) + imageName := "docker-mirror.yizhisec.com/library/redis:alpine" + err = PullImage(ctx, imageName, f) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + // Verify file was created and has content + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s, size: %d bytes", imageName, info.Size()) +} + +// TestPullImage_WithRegistry tests pulling from a specific registry +func TestPullImage_WithRegistry(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "nginx.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Pull from docker.io with explicit registry + imageName := "docker.io/library/nginx:alpine" + err = PullImage(ctx, imageName, f) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s, size: %d bytes", imageName, info.Size()) +} + +// TestPullImage_WithRename tests pulling and renaming an image +func TestPullImage_WithRename(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "renamed.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + imageName := "alpine:3.19" + newName := "my-custom-alpine:latest" + + err = PullImage(ctx, imageName, f, WithRename(newName)) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s and renamed to %s, size: %d bytes", imageName, newName, info.Size()) +} + +// TestPullImage_WithAuth tests pulling with authentication (skip if no credentials) +func TestPullImage_WithAuth(t *testing.T) { + // This test requires actual credentials, so it's skipped by default + t.Skip("Skipping authentication test - requires valid credentials") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "private.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Replace with your private registry details + imageName := "your-registry.com/your-repo/your-image:tag" + username := "your-username" + password := "your-password" + + err = PullImage(ctx, imageName, f, WithAuth(username, password)) + if err != nil { + t.Fatalf("failed to pull private image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled private image %s, size: %d bytes", imageName, info.Size()) +} + +// TestPullImage_WithRetry tests retry functionality +func TestPullImage_WithRetry(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "retry.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Pull with custom retry settings + imageName := "alpine:3.19" + err = PullImage(ctx, imageName, f, + WithRetry(5), // 5 retries + WithRetryDelay(1*time.Second), // 1 second delay + ) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s with retry, size: %d bytes", imageName, info.Size()) +} + +// TestPullImage_WithZeroRetry tests with retry disabled +func TestPullImage_WithZeroRetry(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "no-retry.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Pull with retry disabled + imageName := "alpine:3.19" + err = PullImage(ctx, imageName, f, WithRetry(0)) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s without retry, size: %d bytes", imageName, info.Size()) +} + +// TestPullImage_WithSkipTLSVerify tests pulling with TLS verification disabled +func TestPullImage_WithSkipTLSVerify(t *testing.T) { + // This test is for registries with self-signed certificates + t.Skip("Skipping TLS skip test - requires a registry with self-signed cert") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "insecure.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + imageName := "your-insecure-registry.local/image:tag" + err = PullImage(ctx, imageName, f, WithSkipTLSVerify()) + if err != nil { + t.Fatalf("failed to pull image %s: %v", imageName, err) + } + + if err := f.Close(); err != nil { + t.Fatalf("failed to close file: %v", err) + } + + info, err := os.Stat(outputFile) + if err != nil { + t.Fatalf("failed to stat output file: %v", err) + } + + if info.Size() == 0 { + t.Error("output file is empty") + } + + t.Logf("Successfully pulled %s with TLS verification skipped, size: %d bytes", imageName, info.Size()) +} + +// TestParseImageReference tests the image reference parsing logic +func TestParseImageReference(t *testing.T) { + tests := []struct { + name string + imageName string + wantRegistry string + wantRepository string + wantTag string + wantErr bool + }{ + { + name: "simple image with tag", + imageName: "alpine:3.19", + wantRegistry: "registry-1.docker.io", + wantRepository: "library/alpine", + wantTag: "3.19", + wantErr: false, + }, + { + name: "simple image without tag", + imageName: "alpine", + wantRegistry: "registry-1.docker.io", + wantRepository: "library/alpine", + wantTag: "latest", + wantErr: false, + }, + { + name: "image with namespace", + imageName: "library/nginx:alpine", + wantRegistry: "registry-1.docker.io", + wantRepository: "library/nginx", + wantTag: "alpine", + wantErr: false, + }, + { + name: "custom registry", + imageName: "hub.yizhisec.com/external/alpine:3.22.2", + wantRegistry: "hub.yizhisec.com", + wantRepository: "external/alpine", + wantTag: "3.22.2", + wantErr: false, + }, + { + name: "docker.io explicit", + imageName: "docker.io/library/redis:8.2.2", + wantRegistry: "docker.io", + wantRepository: "library/redis", + wantTag: "8.2.2", + wantErr: false, + }, + { + name: "registry with port", + imageName: "localhost:5000/myimage:v1.0", + wantRegistry: "localhost:5000", + wantRepository: "myimage", + wantTag: "v1.0", + wantErr: false, + }, + { + name: "quay.io registry", + imageName: "quay.io/k0sproject/pause:3.10.1", + wantRegistry: "quay.io", + wantRepository: "k0sproject/pause", + wantTag: "3.10.1", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := parseImageReference(tt.imageName) + if (err != nil) != tt.wantErr { + t.Errorf("parseImageReference() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + + if ref.Registry != tt.wantRegistry { + t.Errorf("parseImageReference() registry = %v, want %v", ref.Registry, tt.wantRegistry) + } + if ref.Repository != tt.wantRepository { + t.Errorf("parseImageReference() repository = %v, want %v", ref.Repository, tt.wantRepository) + } + if ref.Tag != tt.wantTag { + t.Errorf("parseImageReference() tag = %v, want %v", ref.Tag, tt.wantTag) + } + }) + } +} + +// TestPullImage_ContextCancellation tests that context cancellation works +func TestPullImage_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + tmpDir := t.TempDir() + outputFile := filepath.Join(tmpDir, "cancelled.tar") + + f, err := os.Create(outputFile) + if err != nil { + t.Fatalf("failed to create output file: %v", err) + } + defer f.Close() + + // Cancel context immediately + cancel() + + imageName := "alpine:3.19" + err = PullImage(ctx, imageName, f) + if err == nil { + t.Error("expected error due to cancelled context, got nil") + } + + t.Logf("Context cancellation correctly handled: %v", err) +} + +// BenchmarkPullImage benchmarks image pulling performance +func BenchmarkPullImage(b *testing.B) { + ctx := context.Background() + tmpDir := b.TempDir() + + imageName := "alpine:3.19" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + outputFile := filepath.Join(tmpDir, "alpine_bench.tar") + f, err := os.Create(outputFile) + if err != nil { + b.Fatalf("failed to create output file: %v", err) + } + + err = PullImage(ctx, imageName, f) + if err != nil { + b.Fatalf("failed to pull image: %v", err) + } + + f.Close() + os.Remove(outputFile) + } +} diff --git a/pkg/resource/nginx/client.conf b/pkg/resource/nginx/client.conf index b03d151..eb2baef 100644 --- a/pkg/resource/nginx/client.conf +++ b/pkg/resource/nginx/client.conf @@ -33,6 +33,10 @@ server { client_max_body_size 50M; + location /api/v2_2 { + proxy_pass http://u-api-service; + } + location /api/v1/pkg/archive { proxy_pass http://u-api-service/api/v2_2/client/download/check; } @@ -144,12 +148,16 @@ server { client_max_body_size 50M; + location /api/v2_2 { + proxy_pass http://u-api-service; + } + location /api/v1/pkg/archive { proxy_pass http://u-api-service/api/v2_2/client/download/check; } location /api/v1/pkg/archive/version { - proxy_pass http://u-api-service/api/v2_2/client/download/version; + proxy_pass http://u-api-service/api/v2_2/client/version; } location /api/v1/version { diff --git a/pkg/resource/nginx/web.conf b/pkg/resource/nginx/web.conf index ea56fec..c83bc9d 100644 --- a/pkg/resource/nginx/web.conf +++ b/pkg/resource/nginx/web.conf @@ -43,6 +43,10 @@ server { proxy_pass http://u-api-service; } + location /api/v2_2/system { + proxy_pass http://u-api-service; + } + location / { proxy_pass http://front-user-service; } diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go index 359bb65..2f5cdc6 100644 --- a/pkg/resource/resource.go +++ b/pkg/resource/resource.go @@ -5,6 +5,9 @@ import ( ) var ( + //go:embed yaml/registry.yaml + YAMLRegistry string + //go:embed yaml/flannel.yaml YAMLFlannel string diff --git a/pkg/resource/yaml/registry.yaml b/pkg/resource/yaml/registry.yaml new file mode 100644 index 0000000..2a99f5a --- /dev/null +++ b/pkg/resource/yaml/registry.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: db-registry + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: registry-pvc + namespace: db-registry +spec: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: %s + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry + namespace: db-registry +spec: + replicas: 1 + selector: + matchLabels: + app: registry + template: + metadata: + labels: + app: registry + spec: + containers: + - name: registry + image: docker.io/library/registry:2.8.3 + ports: + - containerPort: 5000 + volumeMounts: + - name: data + mountPath: /var/lib/registry + volumes: + - name: data + persistentVolumeClaim: + claimName: registry-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: registry + namespace: db-registry +spec: + clusterIP: 10.96.123.45 + type: NodePork + selector: + app: registry + ports: + - port: 80 + targetPort: 5000 + nodePort: 30500 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry-ui + namespace: db-registry +spec: + replicas: 1 + selector: + matchLabels: + app: registry-ui + template: + metadata: + labels: + app: registry-ui + spec: + containers: + - name: registry + image: docker.io/quiq/registry-ui:0.11.0 + env: + - name: REGISTRY_HOSTNAME + value: "registry:80" + - name: REGISTRY_INSECURE + value: "true" + ports: + - containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: registry-ui + namespace: db-registry +spec: + type: ClusterIP + selector: + app: registry-ui + ports: + - port: 80 + targetPort: 8000 \ No newline at end of file