commit 27fa38aef00ab6e07041158832c911893fb5dac3 Author: zhaoyupeng Date: Mon Nov 24 18:37:44 2025 +0800 🎨 大部分的 make 指令 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f57d7fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.qoder +x-* +dist +.trae +.vscode +.idea \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5efb783 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,47 @@ +# AGENTS.md + +This file provides guidance to Qoder (qoder.com) when working with code in this repository. + +## Project Overview + +This is a Go project (`yizhisec.com/hsv2/forge`) using Go 1.25.2. + +## Development Commands + +### Build +```bash +go build ./... +``` + +### Run Tests +```bash +go test ./... +``` + +### Run Single Test +```bash +go test -run TestName ./path/to/package +``` + +### Lint +```bash +go vet ./... +``` + +### Format Code +```bash +go fmt ./... +``` + +### Tidy Dependencies +```bash +go mod tidy +``` + +## Architecture + +The codebase structure has not yet been established. As the project grows, update this section with: +- Main package organization +- Key architectural patterns +- Important design decisions +- Module relationships diff --git a/README.md b/README.md new file mode 100644 index 0000000..7987e7f --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# hsv2/forge + +一个用于构建与安装 HybridScope v2 相关依赖与资源的命令行工具。基于 Go 与 Cobra 构建，提供统一的 `make` 流程与简单的 `install` 占位命令。 + +## 环境要求 +- Go `1.25.2` +- Linux 环境，能够访问外部下载源（如 `artifactory.yizhisec.com`、`docker.io` 等） + +## 快速开始 +```bash +# 构建 +go build ./... + +# 查看帮助 +./forge --help + +# 进入 make 子命令的帮助 +./forge make --help + +# 示例：准备 Redis 资源 +./forge make redis --replica-count 2 --password ****** +``` + +默认构建目录为 `/root/hsv2-installation`，可通过 `--dir` 修改。 + +## 常用命令 +- `forge`：根命令，支持全局调试开关 `--debug`（参考 `internal/cmd/root.go:12`）。 +- `forge make`：构建依赖与资源（参考 `internal/cmd/make.go:13`）。 + - 全局选项： + - `--dir`：输出基础目录（默认 `/root/hsv2-installation`，参考 `internal/cmd/make.go:40`） + - `--disable-dependency-check`：禁用依赖检查（参考 `internal/cmd/make.go:38`） + - 子命令（节选）： + - `images`：准备镜像相关资源 + - `binaries`：下载并解压 K8s 相关二进制 + - `debs`：下载并解压 Debian 包（Docker） + - `flannel`：生成 Flannel YAML（参考 `internal/cmd/make.flannel.go`） + - `longhorn`：下载 Longhorn chart 并生成 `values.yaml` + - `mysql`：下载 MySQL operator 并生成 `cluster.yaml`，支持 `--replica-count` 与 `--storage-size` + - `redis`：下载 Redis chart 并生成 `values.yaml`，支持 `--replica-count` 与 `--password` + - `emqx`：生成 EMQX 配置 + - `es`：准备 Elasticsearch/Kibana 相关资源 + - `yosguard`：生成 YOSGuard 配置与服务文件 + - `less-dns`：生成 DNS 相关资源 + - `hs-net`：占位命令，当前仅创建目录（参考 `internal/cmd/make.hsnet.go:23`） +- `forge install`：安装占位命令（参考 `internal/cmd/install.go`）。 + +## 目录结构（简要） +- `internal/cmd`：命令行入口与各 `make` 子命令实现 +- `internal/controller/maker`：构建流程控制 +- `pkg/resource`：内置 YAML/SQL/bash 等资源模板 +- `pkg/downloader`、`pkg/archiver`：下载与解压工具 + +## 开发与维护 +- 构建：`go build ./...` +- 测试：`go test ./...` +- 代码检查：`go vet ./...` +- 格式化：`go fmt ./...` +- 依赖清理：`go mod tidy` + +## 注意事项 +- 部分子命令会从公司内网仓库下载资源，请确保网络可达。 +- 如果仅试运行流程，可使用 `--disable-dependency-check` 快速跳过依赖检查。 diff --git a/forge b/forge new file mode 100755 index 0000000..2ee6e09 Binary files /dev/null and b/forge differ diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d25be03 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module yizhisec.com/hsv2/forge + +go 1.25.2 + +require ( + gitea.loveuer.com/yizhisec/pkg3 v0.0.1 + github.com/samber/lo v1.52.0 + github.com/spf13/cobra v1.10.1 +) + +require ( + github.com/fatih/color v1.18.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.22.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8a2e9ee --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +gitea.loveuer.com/yizhisec/pkg3 v0.0.1 h1:bcT58K6W7TQ5u7Lt7B5JxrVbU/riXwcrshd2lu+Q22c= +gitea.loveuer.com/yizhisec/pkg3 v0.0.1/go.mod h1:Ws/tNONjDC4BLdOXAe+wPD6xk3H7TIXvh2de8qKqQpc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= +github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/install.go b/internal/cmd/install.go new file mode 100644 index 0000000..b23312e --- /dev/null +++ b/internal/cmd/install.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func installCmd() *cobra.Command { + _cmd := &cobra.Command{ + Use: "install", + Short: "Install the project", + Long: `Install the built project to the specified location.`, + RunE: func(cmd *cobra.Command, args []string) error { + return runInstall(args) + }, + } + + return _cmd +} + +func runInstall(args []string) error { + fmt.Println("Running install command...") + return nil +} diff --git a/internal/cmd/make.binaries.go b/internal/cmd/make.binaries.go new file mode 100644 index 0000000..a31b96c --- /dev/null +++ b/internal/cmd/make.binaries.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/archiver" +) + +func makeBinaries() *cobra.Command { + cmd := &cobra.Command{ + Use: "binaries", + Aliases: []string{"bin", "B"}, + Short: "Build binary files", + Long: `Build all required binary files for the project.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return os.MkdirAll(filepath.Join(opt.Cfg.Make.Dir, "dependency", "bin"), 0755) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + tarURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/k8s-bin.tar" + binDir = filepath.Join(opt.Cfg.Make.Dir, "dependency") + ) + + logger.Info("开始准备 k8s 二进制文件...") + logger.Debug("下载地址: %s", tarURL) + logger.Debug("目标目录: %s", binDir) + + logger.Info("正在下载二进制文件...") + + if err := archiver.DownloadAndExtract( + cmd.Context(), + tarURL, + binDir, + archiver.WithInsecureSkipVerify(), + archiver.WithGzipCompression(true), + ); err != nil { + logger.Info("❌ 下载并解压二进制文件失败") + return err + } + + logger.Info("✅ 成功下载并解压二进制文件") + return nil + }, + } + + return cmd +} diff --git a/internal/cmd/make.debs.go b/internal/cmd/make.debs.go new file mode 100644 index 0000000..306f46c --- /dev/null +++ b/internal/cmd/make.debs.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/archiver" +) + +func makeDebs() *cobra.Command { + cmd := &cobra.Command{ + Use: "debs", + Aliases: []string{"deb"}, + Short: "Build Debian packages", + Long: `Build all required Debian packages for the project.`, + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Println("Building Debian packages...") + var ( + tarURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/deb/docker.tar.gz" + binDir = filepath.Join(opt.Cfg.Make.Dir, "dependency", "deb") + ) + + logger.Info("开始准备 deb(docker) 文件...") + logger.Debug("下载地址: %s", tarURL) + logger.Debug("目标目录: %s", binDir) + + logger.Info("正在下载二进制文件...") + + if err := archiver.DownloadAndExtract( + cmd.Context(), + tarURL, + binDir, + archiver.WithInsecureSkipVerify(), + archiver.WithGzipCompression(true), + ); err != nil { + logger.Info("❌ 下载并解压 deb(docker) 文件失败") + logger.Debug("下载并解压 deb(docker) 文件失败: %v", err) + return err + } + + logger.Info("✅ 成功下载并解压 deb(docker) 文件") + return nil + }, + } + + return cmd +} diff --git a/internal/cmd/make.emqx.go b/internal/cmd/make.emqx.go new file mode 100644 index 0000000..8567add --- /dev/null +++ b/internal/cmd/make.emqx.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func makeEMQX() *cobra.Command { + _cmd := &cobra.Command{ + Use: "emqx", + Short: "Make EMQX", + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + location = filepath.Join(opt.Cfg.Make.Dir, "emqx") + ) + + logger.Info("MakeEMQX: 开始构建 emqx(mqtt) 依赖...") + + if err = os.MkdirAll(location, 0755); err != nil { + logger.Error("MakeEMQX: 创建 emqx 目录失败") + logger.Debug("MakeEMQX: 创建 emqx 目录失败: %v", err) + return err + } + + if err = os.WriteFile(filepath.Join(location, "emqx.yaml"), resource.YAMLEMQX, 0644); err != nil { + logger.Error("MakeEMQX: 写入 emqx.yaml 失败") + logger.Debug("MakeEMQX: 写入 emqx.yaml 失败, err: %v", err) + return err + } + + logger.Info("MakeEMQX: 构建 emqx(mqtt) 依赖成功!!!") + + return nil + }, + } + + return _cmd +} diff --git a/internal/cmd/make.es.go b/internal/cmd/make.es.go new file mode 100644 index 0000000..ea2b7f8 --- /dev/null +++ b/internal/cmd/make.es.go @@ -0,0 +1,154 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/controller/maker" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/archiver" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func makeES() *cobra.Command { + const ( + dockerFile = ` +FROM docker-mirror.yizhisec.com/library/alpine:3.22.2 + +WORKDIR /data + +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && apk add curl + +ENV TZ=Asia/Shanghai + +COPY plugins /data/plugins +COPY create_index.sh /data/create_index.sh +RUN chmod +x /data/create_index.sh +` + ) + + var ( + makeHelper bool + memRate int + storage int + ) + + _cmd := &cobra.Command{ + Use: "es", + Aliases: []string{"elastic", "elasticsearch"}, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + output []byte + location = filepath.Join(opt.Cfg.Make.Dir, "dependency", "elastic") + helperTarLocation = filepath.Join(opt.Cfg.Make.Dir, "dependency", "image", "es-init-helper.alpine-3.22.2.tar") + ) + + logger.Info("MakeES: 开始构建 elasticsearch(es) 依赖...") + + if err := os.MkdirAll(location, 0755); err != nil { + logger.Error("MakeES: 创建目录失败") + logger.Debug("MakeES: 创建目录失败: %v", err) + return err + } + + if makeHelper { + tmp := filepath.Join(os.TempDir(), "make-es") + logger.Info("MakeES: 重新构建 helper 镜像...(%s)", tmp) + + if err = os.RemoveAll(tmp); err != nil { + logger.Error("MakeES: 删除临时目录失败") + logger.Debug("MakeES: 删除临时目录失败: %v", err) + return err + } + + if err = os.MkdirAll(tmp, 0755); err != nil { + logger.Error("MakeES: 创建临时目录失败") + logger.Debug("MakeES: 创建临时目录失败: %v", err) + return err + } + + if err = os.WriteFile(filepath.Join(tmp, "Dockerfile"), []byte(dockerFile), 0644); err != nil { + logger.Error("MakeES: 写入 Dockerfile 失败") + logger.Debug("MakeES: 写入 Dockerfile 失败: %v", err) + return err + } + + if err = archiver.DownloadAndExtract( + cmd.Context(), + "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/dependency/elasticsearch/v2-es-plugins.tar.gz", + tmp, + archiver.WithInsecureSkipVerify(), + ); err != nil { + logger.Error("MakeES: 下载 es 插件失败") + logger.Debug("MakeES: 下载 es 插件失败: %v", err) + return err + } + + if err = os.WriteFile(filepath.Join(tmp, "create_index.sh"), resource.BashESInit, 0644); err != nil { + logger.Error("MakeES: 写入 create_index.sh 文件失败") + logger.Debug("MakeES: 写入 create_index.sh 文件失败: %v", err) + return err + } + + helpCmd := exec.CommandContext(cmd.Context(), "docker", "build", "-t", + "hub.yizhisec.com/hybridscope/v2/es-init-helper:alpine-3.22.2", + "-f", filepath.Join(tmp, "Dockerfile"), + tmp, + ) + + if output, err = helpCmd.CombinedOutput(); err != nil { + logger.Error("MakeES: 重新构建 helper 镜像失败") + logger.Debug("MakeES: 重新构建 helper 镜像失败: %v", err) + logger.Debug("MakeES: 重新构建 helper 镜像失败: %s", string(output)) + return err + } + + logger.Info("MakeES: 重新构建 helper 镜像成功!!!") + } + + mk := maker.NewMaker() + if err = mk.Image( + cmd.Context(), + "hub.yizhisec.com/hybridscope/v2/es-init-helper:alpine-3.22.2", + maker.WithImageSave(helperTarLocation), + ); err != nil { + logger.Error("MakeES: 准备 es init helper 镜像失败") + logger.Debug("MakeES: 准备 es init helper 镜像失败: %v", err) + return err + } + + bs := []byte(fmt.Sprintf(resource.YAMLES, + memRate, memRate*2, + memRate, memRate, + memRate*2, memRate*2, + storage, + )) + + if err = os.WriteFile(filepath.Join(opt.Cfg.Make.Dir, "dependency", "elastic", "es.yaml"), bs, 0644); err != nil { + logger.Error("MakeES: 写入 es.yaml 文件失败") + logger.Debug("MakeES: 写入 es.yaml 文件失败: %v", err) + return err + } + + if err = os.WriteFile(filepath.Join(opt.Cfg.Make.Dir, "dependency", "elastic", "kibana.yaml"), resource.YAMLKibana, 0644); err != nil { + logger.Error("MakeES: 写入 es.yaml 文件失败") + logger.Debug("MakeES: 写入 es.yaml 文件失败: %v", err) + return err + } + + logger.Info("MakeES: 构建 elastic(es) 依赖成功!!!") + return nil + }, + } + + _cmd.Flags().BoolVar(&makeHelper, "make-helper", false, "重新构建 helper 镜像") + _cmd.Flags().IntVar(&memRate, "mem-rate", 1, "内存倍率(从 1G 开始)") + _cmd.Flags().IntVar(&storage, "storage", 100, "存储空间(单位: G)") + + return _cmd +} diff --git a/internal/cmd/make.flannel.go b/internal/cmd/make.flannel.go new file mode 100644 index 0000000..93a2cf4 --- /dev/null +++ b/internal/cmd/make.flannel.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func makeFlannel() *cobra.Command { + cmd := &cobra.Command{ + Use: "flannel", + Short: "Build Flannel resources", + Long: `Build and prepare Flannel network resources.`, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + location = filepath.Join(opt.Cfg.Make.Dir, "dependency", "flannel") + ) + + logger.Info("MakeFlannel: 开始构建 flannel 资源...") + + logger.Debug("MakeFlannel location: %s", location) + if err = os.Mkdir(location, 0755); err != nil { + logger.Error("MakeFlannel: 创建目录 %s 失败", location) + logger.Debug("MakeFlannel: 创建目录 %s 失败: %v", location, err) + return err + } + + if err = os.WriteFile(filepath.Join(location, "flannel.yaml"), resource.YAMLFlannel, 0644); err != nil { + logger.Error("MakeFlannel: 创建文件 %s 失败", filepath.Join(location, "flannel.yaml")) + logger.Debug("MakeFlannel: 创建文件 %s 失败: %v", filepath.Join(location, "flannel.yaml"), err) + return err + } + + logger.Info("MakeFlannel: 构建 flannel 成功!!!") + + return nil + }, + } + + return cmd +} diff --git a/internal/cmd/make.go b/internal/cmd/make.go new file mode 100644 index 0000000..a09da88 --- /dev/null +++ b/internal/cmd/make.go @@ -0,0 +1,67 @@ +package cmd + +import ( + "os" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/controller/maker" + "yizhisec.com/hsv2/forge/internal/opt" +) + +func makeCmd() *cobra.Command { + + _cmd := &cobra.Command{ + Use: "make", + Short: "Build the project", + Long: `Build the project using the specified configuration.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + ) + + if opt.Cfg.Debug { + logger.SetLogLevel(logger.LogLevelDebug) + logger.Warn("Running in debug mode") + } + + if opt.Cfg.Make.DisableDependencyCheck { + logger.Info("Dependency check disabled") + return nil + } + + mk := maker.NewMaker() + if err = mk.DependencyCheck(cmd.Context()); err != nil { + return err + } + + if err = os.MkdirAll(opt.Cfg.Make.Dir, 0755); err != nil { + return err + } + + logger.Debug("Running make prerun success") + + return nil + }, + } + + _cmd.PersistentFlags().BoolVar(&opt.Cfg.Make.DisableDependencyCheck, "disable-dependency-check", false, "Disable dependency check") + _cmd.PersistentFlags().StringVar(&opt.Cfg.Make.Dir, "dir", "/root/hsv2-installation", "make base directory") + + _cmd.AddCommand( + makeImages(), + makeBinaries(), + makeDebs(), + makeFlannel(), + makeLonghorn(), + makeMysql(), + makeRedis(), + makeES(), + makeEMQX(), + makeYosguard(), + makeLessDNS(), // hs-net dependency + makeHSNet(), + ) + + return _cmd +} diff --git a/internal/cmd/make.hsnet.go b/internal/cmd/make.hsnet.go new file mode 100644 index 0000000..29a2aa5 --- /dev/null +++ b/internal/cmd/make.hsnet.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" +) + +func makeHSNet() *cobra.Command { + _cmd := &cobra.Command{ + Use: "hs-net", + Short: "Build hs-net", + Long: `Build hs-net`, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + location = filepath.Join(opt.Cfg.Make.Dir, "dependency", "hs-net") + ) + + if err = os.MkdirAll(location, 0755); err != nil { + logger.Error("MakeHSNet: 创建目录失败s") + logger.Debug("MakeHSNet: 创建目录失败: %s", err.Error()) + return err + } + + logger.Fatal("MakeHSNet: 构建 hs-net 失败!!!(怎么完善,怎么完善,怎么完善???)") + + return nil + }, + } + + return _cmd +} diff --git a/internal/cmd/make.images.go b/internal/cmd/make.images.go new file mode 100644 index 0000000..4a93c9c --- /dev/null +++ b/internal/cmd/make.images.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "os" + "path/filepath" + + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/controller/maker" + "yizhisec.com/hsv2/forge/internal/opt" +) + +func makeImages() *cobra.Command { + cmd := &cobra.Command{ + Use: "images", + Aliases: []string{"image"}, + Short: "Build and pull Docker images", + Long: `Build and pull all required Docker images for the project.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return os.MkdirAll(filepath.Join(opt.Cfg.Make.Dir, "dependency", "image"), 0755) + }, + RunE: func(cmd *cobra.Command, args []string) error { + type Images struct { + Name string + Fallback string + Save string + Force bool + } + + var images = []*Images{ + {Name: "quay.io/k0sproject/apiserver-network-proxy-agent:v0.32.0", Fallback: "hub.yizhisec.com/external/apiserver-network-proxy-agent:v0.32.0", Save: "k0s.apiserver-network-proxy-agent.tar"}, + {Name: "quay.io/k0sproject/cni-node:1.7.1-k0s.0", Fallback: "", Save: "k0s.cni-node.tar"}, + {Name: "quay.io/k0sproject/coredns:1.12.2", Fallback: "", Save: "k0s.coredns.tar"}, + {Name: "quay.io/k0sproject/kube-proxy:v1.33.4", Fallback: "", Save: "k0s.kube-proxy.tar"}, + {Name: "quay.io/k0sproject/kube-router:v2.5.0-iptables1.8.11-0", Fallback: "", Save: "k0s.kube-router.tar"}, + {Name: "quay.io/k0sproject/metrics-server:v0.7.2-0", Fallback: "", Save: "k0s.metrics-server.tar"}, + {Name: "quay.io/k0sproject/pause:3.10.1", Fallback: "", Save: "k0s.pause.tar"}, + + {Name: "ghcr.io/flannel-io/flannel:v0.27.4", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel:v0.27.4", Save: "flannel.tar"}, + {Name: "ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Fallback: "swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1", Save: "flannel-cni-plugin.tar"}, + + {Name: "docker.io/longhornio/longhorn-engine:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-engine:v1.10.0", Save: "longhorn.longhorn-engine.tar"}, + {Name: "docker.io/longhornio/longhorn-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-manager:v1.10.0", Save: "longhorn.longhorn-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-instance-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-instance-manager:v1.10.0", Save: "longhorn.longhorn-instance-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-share-manager:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-share-manager:v1.10.0", Save: "longhorn.longhorn-share-manager.tar"}, + {Name: "docker.io/longhornio/longhorn-ui:v1.10.0", Fallback: "docker-mirror.yizhisec.com/longhornio/longhorn-ui:v1.10.0", Save: "longhorn.longhorn-ui.tar"}, + {Name: "docker.io/longhornio/csi-snapshotter:v8.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-snapshotter:v8.3.0-20250826", Save: "longhorn.csi-snapshotter.tar"}, + {Name: "docker.io/longhornio/csi-resizer:v1.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-resizer:v1.14.0-20250826", Save: "longhorn.csi-resizer.tar"}, + {Name: "docker.io/longhornio/csi-provisioner:v5.3.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-provisioner:v5.3.0-20250826", Save: "longhorn.csi-provisioner.tar"}, + {Name: "docker.io/longhornio/livenessprobe:v2.16.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/livenessprobe:v2.16.0-20250826", Save: "longhorn.livenessprobe.tar"}, + {Name: "docker.io/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-node-driver-registrar:v2.14.0-20250826", Save: "longhorn.csi-node-driver-registrar.tar"}, + {Name: "docker.io/longhornio/csi-attacher:v4.9.0-20250826", Fallback: "docker-mirror.yizhisec.com/longhornio/csi-attacher:v4.9.0-20250826", Save: "longhorn.csi-attacher.tar"}, + + {Name: "hub.yizhisec.com/external/alpine:3.22.2", Fallback: "", Save: "alpine.tar", Force: true}, + {Name: "hub.yizhisec.com/external/nginx:1.29.1-alpine3.22", Fallback: "", Save: "nginx.1.29.1-alpine3.22.tar"}, + + {Name: "docker.io/bitpoke/mysql-operator:v0.6.3", Fallback: "docker-mirror.yizhisec.com/bitpoke/mysql-operator:v0.6.3", Save: "bitpoke.mysql-operator.v0.6.3.tar"}, + {Name: "docker.io/bitpoke/mysql-operator-orchestrator:v0.6.3", Fallback: "docker-mirror.yizhisec.com/bitpoke/mysql-operator-orchestrator:v0.6.3", Save: "bitpoke.mysql-operator-orchestrator.v0.6.3.tar"}, + {Name: "docker.io/bitpoke/mysql-operator-sidecar-8.0:v0.6.3", Fallback: "docker-mirror.yizhisec.com/bitpoke/mysql-operator-sidecar-8.0:v0.6.3", Save: "bitpoke.mysql-operator-sidecar-8.0.v0.6.3.tar"}, + {Name: "docker.io/library/percona:8.0.28-20", Fallback: "docker-mirror.yizhisec.com/library/percona:8.0.28-20", Save: "percona.8.0.28-20.tar"}, + {Name: "docker.io/prom/mysqld-exporter:v0.13.0", Fallback: "docker-mirror.yizhisec.com/prom/mysqld-exporter:v0.13.0", Save: "prom.mysqld-exporter.v0.13.0.tar"}, + + {Name: "docker.io/bitnami/redis:8.2.2", Fallback: "hub.yizhisec.com/external/bitnami/redis:8.2.2", Save: "bitnami.redis.8.2.2.tar"}, + {Name: "hub.yizhisec.com/external/elasticsearch:7.17.28", Fallback: "", Save: "elasticsearch.7.17.28.tar"}, + {Name: "hub.yizhisec.com/external/kibana:7.17.28", Fallback: "", Save: "kibana.7.17.28.tar"}, + {Name: "hub.yizhisec.com/external/emqx:5.1", Fallback: "", Save: "emqx.5.1.tar"}, + + {Name: "hub.yizhisec.com/build/hybirdscope/front/admin:latest", Fallback: "", Save: "app.front.admin.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/v2/front-user:latest", Fallback: "", Save: "app.front.user.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/user_management:latest", Fallback: "", Save: "app.user.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/gateway_controller:latest", Fallback: "", Save: "app.gateway.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/client_server:latest", Fallback: "", Save: "app.client.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/mie-server:latest", Fallback: "", Save: "app.mie.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/less_dns_service:latest", Fallback: "", Save: "app.less_dns.tar", Force: true}, + {Name: "hub.yizhisec.com/hybridscope/hsnet:release_2.1.0-std", Fallback: "", Save: "app.hsnet.tar", Force: true}, + } + + for _, image := range images { + image.Save = filepath.Join(opt.Cfg.Make.Dir, "dependency", "image", image.Save) + } + + mk := maker.NewMaker() + for _, image := range images { + + opts := []maker.ImageOpt{ + maker.WithImageFallback(image.Fallback), + maker.WithImageSave(image.Save), + } + + if image.Force { + opts = append(opts, maker.WithImageForcePull()) + } + + if err := mk.Image(cmd.Context(), image.Name, opts...); err != nil { + return err + } + } + + return nil + }, + } + + return cmd +} diff --git a/internal/cmd/make.lessdns.go b/internal/cmd/make.lessdns.go new file mode 100644 index 0000000..58bba81 --- /dev/null +++ b/internal/cmd/make.lessdns.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func makeLessDNS() *cobra.Command { + _cmd := &cobra.Command{ + Use: "lessdns", + Short: "Build lessdns", + Long: `Build lessdns`, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + location = filepath.Join(opt.Cfg.Make.Dir, "dependency", "less-dns") + ) + + logger.Info("MakeLessDNS: 开始构建 less-dns...") + if err = os.MkdirAll(location, 0755); err != nil { + logger.Error("MakeLessDNS: 创建目录失败s") + logger.Debug("MakeLessDNS: 创建目录失败: %s", err.Error()) + return err + } + + if err = os.WriteFile(filepath.Join(location, "deployment.yaml"), resource.YAMLLessDNS, 0644); err != nil { + logger.Error("MakeLessDNS: 写入文件失败") + logger.Debug("MakeLessDNS: 写入文件失败: %s", err.Error()) + return err + } + + logger.Info("MakeLessDNS: 构建 less-dns 成功!!!") + + return nil + }, + } + + return _cmd +} diff --git a/internal/cmd/make.longhorn.go b/internal/cmd/make.longhorn.go new file mode 100644 index 0000000..1f9b04b --- /dev/null +++ b/internal/cmd/make.longhorn.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/downloader" +) + +func makeLonghorn() *cobra.Command { + const ( + valuesContent = `# 设置 Longhorn 数据存储路径 +defaultSettings: + defaultDataPath: "/data/longhorn" +# 设置默认副本数 +persistence: + reclaimPolicy: Retain + defaultClassReplicaCount: %d +` + ) + var ( + replicaCount int + ) + + _cmd := &cobra.Command{ + Use: "longhorn", + Short: "Build Longhorn resources", + Long: `Build and prepare Longhorn storage resources.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return os.MkdirAll(filepath.Join(opt.Cfg.Make.Dir, "dependency", "longhorn"), 0755) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + chartURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/charts/longhorn-1.10.0.tgz" + longhornDir = filepath.Join(opt.Cfg.Make.Dir, "dependency", "longhorn") + chartFile = filepath.Join(longhornDir, "longhorn-1.10.0.tgz") + valuesFile = filepath.Join(longhornDir, "values.yaml") + ) + + logger.Info("开始准备 Longhorn 资源...") + logger.Debug("下载地址: %s", chartURL) + logger.Debug("目标目录: %s", longhornDir) + + logger.Info("正在下载 Longhorn chart...") + + // Download the chart file + if err := downloader.Download( + cmd.Context(), + chartURL, + chartFile, + downloader.WithInsecureSkipVerify(), + ); err != nil { + logger.Info("❌ 下载 Longhorn chart 失败") + return err + } + + logger.Info("✅ 成功下载 Longhorn chart") + + // Create values.yaml file + bs := []byte(fmt.Sprintf(valuesContent, replicaCount)) + logger.Debug("创建 values.yaml 文件: %s", valuesFile) + if err := os.WriteFile(valuesFile, bs, 0644); err != nil { + logger.Debug("创建 values.yaml 失败: %v", err) + logger.Info("❌ 创建 values.yaml 失败") + return err + } + + logger.Info("✅ 成功创建 values.yaml") + logger.Debug("Longhorn 资源准备完成") + + return nil + }, + } + + _cmd.Flags().IntVar(&replicaCount, "replica-count", 2, "存储副本数") + + return _cmd +} diff --git a/internal/cmd/make.mysql.go b/internal/cmd/make.mysql.go new file mode 100644 index 0000000..1197b34 --- /dev/null +++ b/internal/cmd/make.mysql.go @@ -0,0 +1,139 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/downloader" +) + +func makeMysql() *cobra.Command { + const ( + _yaml = ` +apiVersion: v1 +kind: Secret +metadata: + name: mysql-secret + namespace: db-mysql +type: Opaque +data: + ROOT_PASSWORD: TDBoTXlzcWwu +--- + +apiVersion: mysql.presslabs.org/v1alpha1 +kind: MysqlCluster +metadata: + name: mysql-cluster + namespace: db-mysql +spec: + mysqlVersion: "8.0" + replicas: %d + image: docker.io/library/percona:8.0.28-20 + secretName: mysql-secret + volumeSpec: + persistentVolumeClaim: + accessModes: ["ReadWriteOnce"] + storageClassName: longhorn + resources: + requests: + storage: %dGi +# podSpec: +# affinity: +# podAntiAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: # 强制规则 +# - labelSelector: +# matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: ["mysql"] +# - key: app.kubernetes.io/instance +# operator: In +# values: ["mysql-cluster"] +# topologyKey: "kubernetes.io/hostname" # 确保不同节点 +# resources: +# requests: +# cpu: "2" +# memory: "2Gi" +# limits: +# cpu: "8" +# memory: "8Gi" +--- + +apiVersion: mysql.presslabs.org/v1alpha1 +kind: MysqlDatabase +metadata: + name: my-database-mie + namespace: db-mysql +spec: + database: mie + clusterRef: + name: mysql-cluster + namespace: db-mysql +` + ) + + var ( + replicas int + storage int + ) + + _cmd := &cobra.Command{ + Use: "mysql", + Short: "Build MySQL resources", + Long: `Build and prepare MySQL database resources.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return os.MkdirAll(filepath.Join(opt.Cfg.Make.Dir, "dependency", "mysql"), 0755) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + chartURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/charts/mysql-operator-0.6.3.tgz" + mysqlDir = filepath.Join(opt.Cfg.Make.Dir, "dependency", "mysql") + chartFile = filepath.Join(mysqlDir, "mysql-operator-0.6.3.tgz") + clusterFile = filepath.Join(mysqlDir, "cluster.yaml") + ) + + logger.Info("开始构建 MySQL 资源...") + logger.Debug("下载地址: %s", chartURL) + logger.Debug("目标目录: %s", mysqlDir) + + // Download MySQL operator chart + logger.Info("正在下载 MySQL operator chart...") + if err := downloader.Download( + cmd.Context(), + chartURL, + chartFile, + downloader.WithInsecureSkipVerify(), + ); err != nil { + logger.Info("❌ 下载 MySQL operator chart 失败") + return err + } + logger.Info("✅ 成功下载 MySQL operator chart") + + bs := []byte(fmt.Sprintf(_yaml, replicas, storage)) + + // Generate secret.yaml + if err = os.WriteFile(clusterFile, bs, 0644); err != nil { + logger.Debug("创建 database.yaml 失败: %v", err) + logger.Info("❌ 创建 database.yaml 失败") + return err + } + + logger.Info("✅ 成功创建 database.yaml") + + logger.Info("✅ MySQL 资源构建成功") + logger.Debug("MySQL 资源准备完成") + + return nil + }, + } + + _cmd.Flags().IntVar(&replicas, "replica-count", 2, "mysql 的副本数") + _cmd.Flags().IntVar(&storage, "storage-size", 50, "mysql 的存储空间") + + return _cmd +} diff --git a/internal/cmd/make.redis.go b/internal/cmd/make.redis.go new file mode 100644 index 0000000..0bc73f8 --- /dev/null +++ b/internal/cmd/make.redis.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/downloader" +) + +func makeRedis() *cobra.Command { + const ( + valuesTemplate = `# Redis configuration +architecture: replication + +image: + registry: docker.io + repository: bitnami/redis + tag: 8.2.2 + pullPolicy: IfNotPresent + +auth: + enabled: true + password: "%s" + +master: + persistence: + enabled: true + storageClass: "longhorn" + size: 5Gi +# resources: +# requests: +# memory: "512Mi" +# cpu: "250m" +# limits: +# memory: "1Gi" +# cpu: "500m" + +replica: + replicaCount: %d + persistence: + enabled: true + storageClass: "longhorn" + size: 5Gi +# resources: +# requests: +# memory: "512Mi" +# cpu: "250m" +# limits: +# memory: "1Gi" +# cpu: "500m" + +metrics: + enabled: false + serviceMonitor: + enabled: false +` + defaultPassword = "HybridScope0xRed1s." + ) + + var ( + replicas int + password string + ) + + _cmd := &cobra.Command{ + Use: "redis", + Short: "Build Redis resources", + Long: `Build and prepare Redis cache resources.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return os.MkdirAll(filepath.Join(opt.Cfg.Make.Dir, "dependency", "redis"), 0755) + }, + RunE: func(cmd *cobra.Command, args []string) error { + var ( + chartURL = "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv3/charts/redis-23.2.2.tgz" + redisDir = filepath.Join(opt.Cfg.Make.Dir, "dependency", "redis") + chartFile = filepath.Join(redisDir, "redis-23.2.2.tgz") + valuesFile = filepath.Join(redisDir, "values.yaml") + ) + + if password == "******" { + password = defaultPassword + } + + logger.Info("开始构建 Redis 资源...") + logger.Debug("下载地址: %s", chartURL) + logger.Debug("目标目录: %s", redisDir) + logger.Debug("Redis 副本数: %d", replicas) + + // Download Redis chart + logger.Info("正在下载 Redis chart...") + if err := downloader.Download( + cmd.Context(), + chartURL, + chartFile, + downloader.WithInsecureSkipVerify(), + ); err != nil { + logger.Info("❌ 下载 Redis chart 失败") + return err + } + logger.Info("✅ 成功下载 Redis chart") + + // Generate values.yaml + logger.Debug("创建 values.yaml: %s", valuesFile) + valuesContent := fmt.Sprintf(valuesTemplate, password, replicas) + if err := os.WriteFile(valuesFile, []byte(valuesContent), 0644); err != nil { + logger.Debug("创建 values.yaml 失败: %v", err) + logger.Info("❌ 创建 values.yaml 失败") + return err + } + logger.Info("✅ 成功创建 values.yaml") + + logger.Info("✅ Redis 资源构建成功") + logger.Debug("Redis 资源准备完成") + + return nil + }, + } + + _cmd.Flags().IntVar(&replicas, "replica-count", 2, "Redis 副本数") + _cmd.Flags().StringVar(&password, "password", "******", "Redis 密码") + + return _cmd +} diff --git a/internal/cmd/make.yosguard.go b/internal/cmd/make.yosguard.go new file mode 100644 index 0000000..5cd03c5 --- /dev/null +++ b/internal/cmd/make.yosguard.go @@ -0,0 +1,120 @@ +package cmd + +import ( + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" + "yizhisec.com/hsv2/forge/pkg/downloader" + "yizhisec.com/hsv2/forge/pkg/resource" +) + +func makeYosguard() *cobra.Command { + const ( + _configTemplate = ` +Web: + # default listen in docker0 + Host: 172.17.0.1 + Port: 7788 + +UUIDFilePath: /etc/yosguard/uuid + +# 心跳间隔: 单位秒,默认为5 +HeartbeatDuration: 5 + +# 控制器 yosguard 地址 +ControllerServer: + Host: dasheng.zhsftech.debug + Port: 443 + +# True: 作为控制器运行; False: 不作为控制器运行 +AsController: true + +# True: 作为网关运行; False: 不作为网关运行 +AsGateway: false + +Database: + SQLite: + DBPath: "/etc/yosguard/db/yosguard.db" + SQLPath: "/etc/yosguard/db/create.sql"` + _service = ` +[Unit] +Description=YiZhiSec YOSGuard +After=network.target + +[Service] +Type=simple +User=root +ExecStart=/usr/local/bin/yosguard web --host __host__ +StandardOutput=journal +StandardError=journal +Nice=-20 +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target` + ) + _cmd := &cobra.Command{ + Use: "yosguard", + Aliases: []string{"YOS"}, + Short: "Make Yosguard", + RunE: func(cmd *cobra.Command, args []string) error { + var ( + err error + location = filepath.Join(opt.Cfg.Make.Dir, "dependency", "yosguard") + ) + + logger.Info("MakeYosguard: 开始构建 yosguard...") + + if err = os.MkdirAll(location, 0755); err != nil { + logger.Error("MakeYosguard: 创建 yosguard 目录失败") + logger.Debug("MakeYosguard: 创建 yosguard 目录失败: %v", err) + return err + } + + // 1. download yosguard bin: https://artifactory.yizhisec.com:443/artifactory/filestore/hsv2/bin/yosguard + _url := "https://artifactory.yizhisec.com:443/artifactory/filestore/hsv2/bin/yosguard" + logger.Debug("MakeYosguard: start download bin from %s", _url) + if err = downloader.Download( + cmd.Context(), + _url, + filepath.Join(location, "yosguard"), + downloader.WithInsecureSkipVerify(), + downloader.WithFileMode(0755), + ); err != nil { + logger.Error("MakeYosguard: 下载 yosguard 失败") + logger.Debug("MakeYosguard: 下载 yosguard 失败: %v", err) + return err + } + + // 2. generate config_template.yml + if err = os.WriteFile(filepath.Join(location, "config_template.yml"), []byte(_configTemplate), 0644); err != nil { + logger.Error("MakeYosguard: 生成 config_template.yml 失败") + logger.Debug("MakeYosguard: 生成 config_template.yml 失败: %v", err) + return err + } + + // 3. generate create.sql + if err = os.WriteFile(filepath.Join(location, "create.sql"), resource.SQLYosguard, 0644); err != nil { + logger.Error("MakeYosguard: 生成 create.sql 失败") + logger.Debug("MakeYosguard: 生成 create.sql 失败: %v", err) + return err + } + + // 4. generate systemd file + if err = os.WriteFile(filepath.Join(location, "yosguard.service"), []byte(_service), 0644); err != nil { + logger.Error("MakeYosguard: 生成 yosguard.service 失败") + logger.Debug("MakeYosguard: 生成 yosguard.service 失败: %v", err) + return err + } + + logger.Info("MakeYosguard: 构建 yosguard成功!!!") + return nil + }, + } + + return _cmd +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go new file mode 100644 index 0000000..bb4148f --- /dev/null +++ b/internal/cmd/root.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "context" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/spf13/cobra" + "yizhisec.com/hsv2/forge/internal/opt" +) + +var rootCmd = &cobra.Command{ + Use: "forge", + Short: "Forge is a tool for building and installing hsv2", + Long: `A tool for managing build and installation workflows for hsv2.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if opt.Cfg.Debug { + logger.SetLogLevel(logger.LogLevelDebug) + logger.Warn("running in debug mode") + } + + return nil + }, +} + +func Execute(ctx context.Context) error { + return rootCmd.ExecuteContext(ctx) +} + +func init() { + + rootCmd.PersistentFlags().BoolVar(&opt.Cfg.Debug, "debug", false, "Enable debug mode") + + rootCmd.AddCommand( + makeCmd(), + installCmd(), + ) +} diff --git a/internal/controller/maker/check.go b/internal/controller/maker/check.go new file mode 100644 index 0000000..0cccf9d --- /dev/null +++ b/internal/controller/maker/check.go @@ -0,0 +1,35 @@ +package maker + +import ( + "context" + "fmt" + "os/exec" +) + +func (m *maker) DependencyCheck(ctx context.Context) error { + // 1. test wget command + if err := checkCommand(ctx, "wget", "--version"); err != nil { + return fmt.Errorf("wget 命令未找到: %w", err) + } + + // 2. test tar command + if err := checkCommand(ctx, "tar", "--version"); err != nil { + return fmt.Errorf("tar 命令未找到: %w", err) + } + + // 3. test docker command + if err := checkCommand(ctx, "docker", "info"); err != nil { + return fmt.Errorf("docker 命令未找到: %w", err) + } + + return nil +} + +// checkCommand checks if a command is available by running it with the specified args +func checkCommand(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + if err := cmd.Run(); err != nil { + return err + } + return nil +} diff --git a/internal/controller/maker/image.go b/internal/controller/maker/image.go new file mode 100644 index 0000000..517122e --- /dev/null +++ b/internal/controller/maker/image.go @@ -0,0 +1,102 @@ +package maker + +import ( + "context" + "os/exec" + + "gitea.loveuer.com/yizhisec/pkg3/logger" + "github.com/samber/lo" +) + +type imageOpt struct { + Fallbacks []string + IgnoreFailure bool + ForcePull bool + Save string +} + +type ImageOpt func(*imageOpt) + +func WithImageFallback(fallbacks ...string) ImageOpt { + return func(o *imageOpt) { + o.Fallbacks = lo.Filter(fallbacks, func(item string, _ int) bool { return item != "" }) + } +} + +func WithImageSave(filename string) ImageOpt { + return func(o *imageOpt) { + o.Save = filename + } +} + +func WithImageForcePull() ImageOpt { + return func(o *imageOpt) { + o.ForcePull = true + } +} + +func (m *maker) Image(ctx context.Context, name string, opts ...ImageOpt) error { + logger.Info("开始获取镜像: %s", name) + var ( + err error + o = &imageOpt{} + _cmd *exec.Cmd + ) + + for _, fn := range opts { + fn(o) + } + + logger.Debug("maker.Image: name = %s, opt = %#v", name, o) + + if !o.ForcePull { + _cmd = exec.CommandContext(ctx, "docker", "image", "inspect", name) + if err = _cmd.Run(); err == nil { + logger.Info("💾 镜像 %s 已存在", name) + goto SAVE + } + } + + _cmd = exec.CommandContext(ctx, "docker", "pull", name) + if err = _cmd.Run(); err != nil { + logger.Debug("获取原始镜像 %s 失败： %v", name, err) + } else { + logger.Info("✅ 成功获取镜像: %s", name) + goto SAVE + } + + for _, fallback := range o.Fallbacks { + logger.Info("开始获取镜像: %s (%s)", name, fallback) + _cmd := exec.CommandContext(ctx, "docker", "pull", fallback) + if err = _cmd.Run(); err != nil { + logger.Debug("获取镜像 %s (%s) 失败: %v", name, fallback, err) + continue + } + + // pull success, retag image + _cmd = exec.CommandContext(ctx, "docker", "tag", fallback, name) + if err = _cmd.Run(); err != nil { + logger.Debug("重命名镜像 %s => %s 失败: %s", fallback, name, err) + continue + } + + logger.Info("✅ 成功获取镜像: %s (%s)", name, fallback) + break + } + +SAVE: + if o.Save != "" { + logger.Debug("保存镜像 %s 到 %s", name, o.Save) + if err = exec.CommandContext(ctx, "docker", "save", "-o", o.Save, name).Run(); err != nil { + logger.Debug("保存镜像 %s 到 %s 失败: %v", name, o.Save, err) + } else { + logger.Info("✅ 镜像 %s 保存到 %s 成功", name, o.Save) + } + } + + if o.IgnoreFailure { + return nil + } + + return err +} diff --git a/internal/controller/maker/image_test.go b/internal/controller/maker/image_test.go new file mode 100644 index 0000000..3af73d8 --- /dev/null +++ b/internal/controller/maker/image_test.go @@ -0,0 +1,16 @@ +package maker + +import ( + "testing" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +func TestImage(t *testing.T) { + logger.SetLogLevel(logger.LogLevelDebug) + m := NewMaker() + m.Image(t.Context(), "docker.io/nginx:1.29.3-alpine3.22", + WithImageFallback("swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/nginx:1.29.3-alpine3.22"), + WithImageSave("/root/nginx.tar"), + ) +} diff --git a/internal/controller/maker/maker.go b/internal/controller/maker/maker.go new file mode 100644 index 0000000..7c02070 --- /dev/null +++ b/internal/controller/maker/maker.go @@ -0,0 +1,7 @@ +package maker + +type maker struct{} + +func NewMaker() *maker { + return &maker{} +} diff --git a/internal/opt/opt.go b/internal/opt/opt.go new file mode 100644 index 0000000..e2c5f52 --- /dev/null +++ b/internal/opt/opt.go @@ -0,0 +1,13 @@ +package opt + +type config struct { + Debug bool + Make struct { + DisableDependencyCheck bool + Dir string + } +} + +var ( + Cfg = &config{} +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..862355b --- /dev/null +++ b/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "yizhisec.com/hsv2/forge/internal/cmd" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + if err := cmd.Execute(ctx); err != nil { + os.Exit(1) + } +} diff --git a/pkg/archiver/archiver.go b/pkg/archiver/archiver.go new file mode 100644 index 0000000..04441ab --- /dev/null +++ b/pkg/archiver/archiver.go @@ -0,0 +1,225 @@ +package archiver + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +// Options defines options for downloading and extracting archives +type Options struct { + // InsecureSkipVerify skips TLS certificate verification (equivalent to wget --no-check-certificate) + InsecureSkipVerify bool + // HTTPClient allows providing a custom HTTP client + HTTPClient *http.Client + // OnProgress is called during extraction with the current file being extracted + OnProgress func(filename string, index int, total int) + // IsGzipped explicitly specifies whether the archive is gzip compressed + // If nil, auto-detect based on file extension (.tar.gz, .tgz) + IsGzipped *bool +} + +// Option is a functional option for configuring the archiver +type Option func(*Options) + +// WithInsecureSkipVerify skips TLS certificate verification +func WithInsecureSkipVerify() Option { + return func(o *Options) { + o.InsecureSkipVerify = true + } +} + +// WithHTTPClient sets a custom HTTP client +func WithHTTPClient(client *http.Client) Option { + return func(o *Options) { + o.HTTPClient = client + } +} + +// WithProgress sets a progress callback +func WithProgress(callback func(filename string, index int, total int)) Option { + return func(o *Options) { + o.OnProgress = callback + } +} + +// WithGzipCompression explicitly sets whether the archive is gzip compressed +func WithGzipCompression(isGzipped bool) Option { + return func(o *Options) { + o.IsGzipped = &isGzipped + } +} + +// DownloadAndExtract downloads a tar or tar.gz file from URL and extracts it to destDir +// Supports both .tar and .tar.gz formats +func DownloadAndExtract(ctx context.Context, url, destDir string, opts ...Option) error { + options := &Options{ + InsecureSkipVerify: false, + } + + for _, opt := range opts { + opt(options) + } + + logger.Debug("开始下载和解压: %s -> %s", url, destDir) + + // Create HTTP client + client := options.HTTPClient + if client == nil { + client = &http.Client{} + if options.InsecureSkipVerify { + logger.Debug("TLS 证书验证已禁用") + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + } + + // Download the file + logger.Debug("发起 HTTP 请求: %s", url) + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + logger.Debug("创建请求失败: %v", err) + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + logger.Debug("下载失败: %v", err) + return fmt.Errorf("failed to download: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Debug("HTTP 状态码异常: %d", resp.StatusCode) + return fmt.Errorf("bad status: %s", resp.Status) + } + + logger.Debug("下载成功，准备解压") + + // Determine if the file is gzipped + var reader io.Reader = resp.Body + var isGzipped bool + + if options.IsGzipped != nil { + // Explicitly specified by option + isGzipped = *options.IsGzipped + logger.Debug("压缩格式由选项指定: isGzipped=%v", isGzipped) + } else { + // Auto-detect based on file extension + isGzipped = strings.HasSuffix(url, ".tar.gz") || strings.HasSuffix(url, ".tgz") + logger.Debug("根据文件扩展名自动检测压缩格式: isGzipped=%v", isGzipped) + } + + if isGzipped { + logger.Debug("使用 gzip 解压") + gzReader, err := gzip.NewReader(resp.Body) + if err != nil { + logger.Debug("创建 gzip reader 失败: %v", err) + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzReader.Close() + reader = gzReader + } else { + logger.Debug("tar 格式（未压缩）") + } + + // Extract tar archive + if err := extractTar(reader, destDir, options); err != nil { + return fmt.Errorf("failed to extract tar: %w", err) + } + + logger.Debug("解压完成") + return nil +} + +// extractTar extracts a tar archive to the destination directory +func extractTar(r io.Reader, destDir string, options *Options) error { + tarReader := tar.NewReader(r) + fileCount := 0 + totalFiles := 0 + + // First pass: count total files (optional, for progress reporting) + // For now, we'll just extract directly + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + logger.Debug("读取 tar 条目失败: %v", err) + return fmt.Errorf("failed to read tar header: %w", err) + } + + target := filepath.Join(destDir, header.Name) + logger.Debug("解压文件: %s", header.Name) + + // Call progress callback if provided + if options.OnProgress != nil { + options.OnProgress(header.Name, fileCount, totalFiles) + } + + switch header.Typeflag { + case tar.TypeDir: + // Create directory + if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { + logger.Debug("创建目录失败 %s: %v", target, err) + return fmt.Errorf("failed to create directory %s: %w", target, err) + } + + case tar.TypeReg: + // Create regular file + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + logger.Debug("创建父目录失败 %s: %v", filepath.Dir(target), err) + return fmt.Errorf("failed to create parent directory: %w", err) + } + + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(header.Mode)) + if err != nil { + logger.Debug("创建文件失败 %s: %v", target, err) + return fmt.Errorf("failed to create file %s: %w", target, err) + } + + if _, err := io.Copy(outFile, tarReader); err != nil { + outFile.Close() + logger.Debug("写入文件失败 %s: %v", target, err) + return fmt.Errorf("failed to write file %s: %w", target, err) + } + outFile.Close() + fileCount++ + + case tar.TypeSymlink: + // Create symbolic link + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + logger.Debug("创建符号链接父目录失败 %s: %v", filepath.Dir(target), err) + return fmt.Errorf("failed to create parent directory for symlink: %w", err) + } + + // Remove existing file/link if exists + os.Remove(target) + + if err := os.Symlink(header.Linkname, target); err != nil { + logger.Debug("创建符号链接失败 %s -> %s: %v", target, header.Linkname, err) + return fmt.Errorf("failed to create symlink %s -> %s: %w", target, header.Linkname, err) + } + fileCount++ + + default: + logger.Debug("跳过不支持的文件类型: %s (type: %v)", header.Name, header.Typeflag) + } + } + + logger.Debug("解压完成，共 %d 个文件", fileCount) + return nil +} diff --git a/pkg/archiver/archiver_test.go b/pkg/archiver/archiver_test.go new file mode 100644 index 0000000..665e62d --- /dev/null +++ b/pkg/archiver/archiver_test.go @@ -0,0 +1,47 @@ +package archiver + +import ( + "context" + "testing" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +func TestDownloadAndExtract(t *testing.T) { + logger.SetLogLevel(logger.LogLevelDebug) + + ctx := context.Background() + + // Example: download and extract a tar.gz file + err := DownloadAndExtract( + ctx, + "https://example.com/archive.tar.gz", + "/tmp/test-extract", + WithInsecureSkipVerify(), + ) + + if err != nil { + t.Logf("Expected error for test URL: %v", err) + } +} + +func TestDownloadAndExtractWithProgress(t *testing.T) { + logger.SetLogLevel(logger.LogLevelDebug) + + ctx := context.Background() + + // Example: with progress callback + err := DownloadAndExtract( + ctx, + "https://example.com/archive.tar", + "/tmp/test-extract", + WithInsecureSkipVerify(), + WithProgress(func(filename string, index int, total int) { + t.Logf("Extracting: %s", filename) + }), + ) + + if err != nil { + t.Logf("Expected error for test URL: %v", err) + } +} diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go new file mode 100644 index 0000000..8e8c6d1 --- /dev/null +++ b/pkg/downloader/downloader.go @@ -0,0 +1,195 @@ +package downloader + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "gitea.loveuer.com/yizhisec/pkg3/logger" +) + +// Options defines options for downloading files +type Options struct { + // InsecureSkipVerify skips TLS certificate verification + InsecureSkipVerify bool + // HTTPClient allows providing a custom HTTP client + HTTPClient *http.Client + // OnProgress is called during download with bytes downloaded and total size + OnProgress func(downloaded, total int64) + // CreateDirs automatically creates parent directories if they don't exist + CreateDirs bool + // Overwrite allows overwriting existing files + Overwrite bool + FileMode os.FileMode +} + +// Option is a functional option for configuring the downloader +type Option func(*Options) + +// WithInsecureSkipVerify skips TLS certificate verification +func WithInsecureSkipVerify() Option { + return func(o *Options) { + o.InsecureSkipVerify = true + } +} + +// WithHTTPClient sets a custom HTTP client +func WithHTTPClient(client *http.Client) Option { + return func(o *Options) { + o.HTTPClient = client + } +} + +// WithProgress sets a progress callback +func WithProgress(callback func(downloaded, total int64)) Option { + return func(o *Options) { + o.OnProgress = callback + } +} + +// WithoutCreateDirs disables automatic creation of parent directories +func WithoutCreateDirs() Option { + return func(o *Options) { + o.CreateDirs = false + } +} + +// WithoutOverwrite prevents overwriting existing files +func WithoutOverwrite() Option { + return func(o *Options) { + o.Overwrite = false + } +} + +func WithFileMode(mode os.FileMode) Option { + return func(o *Options) { + o.FileMode = mode + } +} + +// Download downloads a file from URL to the specified destination +func Download(ctx context.Context, url, dest string, opts ...Option) error { + options := &Options{ + InsecureSkipVerify: false, + CreateDirs: true, + Overwrite: true, + FileMode: 0644, + } + + for _, opt := range opts { + opt(options) + } + + logger.Debug("开始下载文件: %s -> %s", url, dest) + + // Check if file exists and overwrite is disabled + if !options.Overwrite { + if _, err := os.Stat(dest); err == nil { + logger.Debug("文件已存在且不允许覆盖: %s", dest) + return fmt.Errorf("file already exists: %s", dest) + } + } + + // Create parent directories if needed + if options.CreateDirs { + dir := filepath.Dir(dest) + if err := os.MkdirAll(dir, 0755); err != nil { + logger.Debug("创建目录失败 %s: %v", dir, err) + return fmt.Errorf("failed to create directory: %w", err) + } + } + + // Create HTTP client + client := options.HTTPClient + if client == nil { + client = &http.Client{} + if options.InsecureSkipVerify { + logger.Debug("TLS 证书验证已禁用") + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + } + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + logger.Debug("创建请求失败: %v", err) + return fmt.Errorf("failed to create request: %w", err) + } + + // Execute request + logger.Debug("发起 HTTP 请求: %s", url) + resp, err := client.Do(req) + if err != nil { + logger.Debug("下载失败: %v", err) + return fmt.Errorf("failed to download: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Debug("HTTP 状态码异常: %d", resp.StatusCode) + return fmt.Errorf("bad status: %s", resp.Status) + } + + // Get content length for progress reporting + contentLength := resp.ContentLength + logger.Debug("文件大小: %d bytes", contentLength) + + outFile, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, options.FileMode) + if err != nil { + logger.Debug("创建文件失败 %s: %v", dest, err) + return fmt.Errorf("failed to create file: %w", err) + } + defer outFile.Close() + + // Copy content with optional progress reporting + var written int64 + if options.OnProgress != nil && contentLength > 0 { + // Use progress reader + reader := &progressReader{ + reader: resp.Body, + callback: options.OnProgress, + total: contentLength, + } + written, err = io.Copy(outFile, reader) + } else { + written, err = io.Copy(outFile, resp.Body) + } + + if err != nil { + logger.Debug("写入文件失败 %s: %v", dest, err) + return fmt.Errorf("failed to write file: %w", err) + } + + if options.FileMode != 0 { + if err := os.Chmod(dest, options.FileMode); err != nil { + logger.Debug("设置文件权限失败 %s: %v", dest, err) + return fmt.Errorf("failed to set file mode: %w", err) + } + } + + logger.Debug("文件下载成功: %s (%d bytes)", dest, written) + return nil +} + +// progressReader wraps an io.Reader to report progress +type progressReader struct { + reader io.Reader + callback func(downloaded, total int64) + total int64 + downloaded int64 +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.downloaded += int64(n) + if pr.callback != nil { + pr.callback(pr.downloaded, pr.total) + } + return n, err +} diff --git a/pkg/resource/emqx.yaml b/pkg/resource/emqx.yaml new file mode 100644 index 0000000..e99741a --- /dev/null +++ b/pkg/resource/emqx.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: db-emqx +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emqx + namespace: db-emqx +spec: + replicas: 1 + selector: + matchLabels: + app: emqx + template: + metadata: + labels: + app: emqx + spec: + containers: + - name: emqx + image: hub.yizhisec.com/external/emqx:5.1 + ports: + - containerPort: 1883 + name: mqtt + - containerPort: 8883 + name: mqtt-ssl + - containerPort: 18083 + name: dashboard + - containerPort: 18084 + name: websocket + env: + - name: EMQX_NODE_NAME + value: "emqx@single-node" + - name: EMQX_DASHBOARD__DEFAULT_PASSWORD + value: "YizhiSEC@123" + +--- +apiVersion: v1 +kind: Service +metadata: + name: emqx-service + namespace: db-emqx +spec: + selector: + app: emqx + type: ClusterIP + ports: + - name: mqtt + port: 1883 + targetPort: 1883 + - name: mqtt-ssl + port: 8883 + targetPort: 8883 + - name: dashboard + port: 18083 + targetPort: 18083 + - name: websocket + port: 18084 + targetPort: 18084 + diff --git a/pkg/resource/es.init.sh b/pkg/resource/es.init.sh new file mode 100644 index 0000000..190a8cd --- /dev/null +++ b/pkg/resource/es.init.sh @@ -0,0 +1,1817 @@ +#!/bin/bash + +set -x + +while [[ "$(curl -s -o /dev/null -w '%{http_code}' es-service:9200)" != "200" ]]; do sleep 1; done + +ES_URL="es-service:9200/" +JSON_HEADER="Content-Type: application/json" + +function put_es_with_retry() { + wait=0 + timeout_msg="timeout_msg" + while [ -n "$timeout_msg" ] && [ $wait -le 3 ]; do + timeout_msg=$(curl -s -X PUT $ES_URL$1 -H "$JSON_HEADER" -d "$2" | grep "process_cluster_event_timeout_exception") + + if [ -n "$timeout_msg" ]; then + wait=$((wait + 1)) + sleep 300 + fi + done +} + +function put_es() { + curl -X PUT $ES_URL$1 -H "$JSON_HEADER" -d "$2" +} + +# es初始启动几分钟内，请求创建index可能会失败，process_cluster_event_timeout_exception；第一个请求检查一下。 +put_es_with_retry "app_block_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "domain_id": { + "type": "short" + }, + "ip": { + "type": "ip" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "keyword" + }, + "user_avatar": { + "type": "keyword" + }, + "process_name": { + "type": "keyword" + }, + "process_path": { + "type": "keyword" + }, + "md5": { + "type": "keyword" + }, + "device_uuid": { + "type": "keyword" + }, + "device_name": { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "access_mode": { + "type": "byte" + } + } + } +} +' + +put_es "network_log" ' +{ + "settings": { + "analysis": { + "analyzer": { + "keyword_lowercase": { + "tokenizer": "keyword", + "filter": [ "lowercase" ] + } + } + } + }, + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + }, + "security_space_name": { + "type": "keyword" + }, + "domain_id": { + "type": "integer" + }, + "local_ip": { + "type": "ip" + }, + "process_name": { + "type": "keyword", + "fields": { + "suggest": { + "type": "completion", + "analyzer": "keyword_lowercase" + } + } + }, + "target_address": { + "type": "text" + }, + "target_port": { + "type": "integer" + }, + "timestamp": { + "type": "long" + }, + "target_address_key": { + "type": "keyword", + "fields": { + "suggest": { + "type": "completion", + "analyzer": "keyword_lowercase" + } + } + }, + "target_time": { + "type": "date" + }, + "reason": { + "type": "keyword" + }, + "result": { + "type": "boolean" + }, + "access_type": { + "type": "integer" + } + } + } +} +' + +put_es "data_delivery_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "data": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "data_id": { + "type": "long" + }, + "delivery_type": { + "type": "long" + }, + "dest_user_id": { + "type": "long" + }, + "destination": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "dev_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "dev_uuid": { + "type": "keyword" + }, + "display_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "domain_id": { + "type": "long" + }, + "security_domain": { + "type": "keyword" + }, + "share_id": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "user_avatar": { + "type": "keyword" + }, + "ip": { + "type": "keyword" + }, + "is_folder": { + "type": "boolean" + }, + "file_size": { + "type": "long" + }, + "file_sha256": { + "type": "keyword" + }, + "action": { + "type": "byte" + }, + "success": { + "type": "boolean" + }, + "file_count": { + "type": "integer" + }, + "warning_reason": { + "type": "keyword" + }, + "reject_reason": { + "type": "keyword" + }, + "is_warning": { + "type": "boolean" + }, + "is_rejected": { + "type": "boolean" + }, + "application_id": { + "type": "long" + }, + "access_mode": { + "type": "byte" + }, + "device_type": { + "type": "byte" + }, + "is_domain_interior": { + "type": "boolean" + }, + "file_sha256s": { + "type": "keyword" + }, + "process_name": { + "type": "keyword" + }, + "process_hash": { + "type": "keyword" + }, + "destination_address": { + "type": "keyword" + } + } + } +} +' + +put_es "operation_log" ' +{ + "mappings" : { + "properties" : { + "tenant_id": { + "type": "short" + }, + "detail" : { + "type" : "text" + }, + "op_time" : { + "type" : "long" + }, + "op_type" : { + "type" : "long" + }, + "operator_id" : { + "type" : "long" + }, + "operator_ip" : { + "type" : "keyword" + }, + "operator_name" : { + "type" : "keyword" + } + } + } +} +' + +put_es "device_detail_info" ' +{ + "mappings" : { + "properties" : { + "dev_uuid" : { + "type" : "keyword" + }, + "update_time" : { + "type" : "long" + }, + "antivirus": { + "type": "text", + "index": false + }, + "patch": { + "type": "text", + "index": false + }, + "software": { + "type": "text", + "index": false + }, + "system_status": { + "type": "text", + "index": false + }, + "process": { + "type": "text", + "index": false + }, + "service": { + "type": "text", + "index": false + }, + "network": { + "type": "text", + "index": false + }, + "startup": { + "type": "text", + "index": false + } + } + } +}' + +put_es "rule_file_op_log" ' +{ + "mappings": { + "properties": { + "share_id": { + "type": "long" + }, + "share_log_id": { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "log_type": { + "type": "long" + }, + "content": { + "type": "text", + "index": false + } + } + } +} +' + +put_es "delete_domain_user_data_log" ' +{ + "mappings" : { + "properties" : { + "tenant_id": { + "type": "short" + }, + "domain_id" : { + "type" : "short" + }, + "user_id" : { + "type" : "long" + }, + "user_name" : { + "type" : "text" + }, + "delete_status" : { + "type" : "text" + }, + "timestamp" : { + "type" : "long" + }, + "detail" : { + "type" : "object" + } + } + } +} +' + +put_es "delete_user_data_status_log" ' +{ + "mappings" : { + "properties" : { + "tenant_id": { + "type": "short" + }, + "domain_id" : { + "type" : "short" + }, + "user_id" : { + "type" : "long" + }, + "device_uuid" : { + "type" : "text" + }, + "deleted" : { + "type" : "boolean" + }, + "timestamp" : { + "type" : "long" + } + } + } +} +' + +put_es "dynamic_auth_log" ' +{ + "mappings": { + "properties": { + "ip": { + "type": "ip" + }, + "user_id": { + "type": "long" + }, + "device_uuid": { + "type": "keyword" + }, + "reason": { + "type": "integer" + }, + "msg": { + "type": "text" + }, + "handle_type": { + "type": "integer" + }, + "handle_score": { + "type": "integer" + }, + "timestamp": { + "type": "long" + }, + "tenant_id": { + "type": "short" + } + } + } +} +' + +put_es "client_auth_log" ' +{ + "mappings": { + "properties": { + "user_id": { + "type": "long" + }, + "user_name": { + "type": "keyword" + }, + "device_uuid": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "result": { + "type": "boolean" + }, + "msg": { + "type": "text" + }, + "timestamp": { + "type": "long" + }, + "tenant_id": { + "type": "short" + }, + "access_mode": { + "type": "byte" + } + } + } +} +' + +put_es "clipboard_log" ' +{ + "mappings" : { + "properties" : { + "tenant_id" : { + "type" : "short" + }, + "domain_id" : { + "type" : "short" + }, + "ip": { + "type": "ip" + }, + "direction" : { + "type" : "byte" + }, + "user_id" : { + "type" : "long" + }, + "user_name" : { + "type" : "keyword" + }, + "user_avatar" : { + "type" : "keyword", + "index": false + }, + "device_uuid" : { + "type" : "keyword" + }, + "device_name" : { + "type" : "keyword" + }, + "is_rejected" : { + "type" : "boolean" + }, + "data_type" : { + "type" : "byte" + }, + "content" : { + "type" : "keyword", + "index": false + }, + "image_ext" : { + "type" : "keyword" + }, + "timestamp" : { + "type" : "long" + }, + "access_mode": { + "type": "byte" + } + } + } +} +' + +put_es "client_integrity_exception_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_uuid": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "timestamp": { + "type": "long" + }, + "version": { + "type": "keyword" + }, + "type": { + "type": "integer" + }, + "content": { + "type": "text" + }, + "action": { + "type": "integer" + }, + "file_sha256": { + "type": "keyword" + }, + "file_name": { + "type": "keyword" + }, + "file_size": { + "type": "long" + } + } + } +} +' + +put_es "client_process_protect_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_uuid": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "timestamp": { + "type": "long" + }, + "version": { + "type": "keyword" + }, + "protect_process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "reject_process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "username": { + "type": "keyword" + }, + "action": { + "type": "integer" + }, + "success": { + "type": "boolean" + } + } + } +} +' + +put_es "client_install_directory_protect_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_uuid": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "timestamp": { + "type": "long" + }, + "version": { + "type": "keyword" + }, + "process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "username": { + "type": "keyword" + }, + "action": { + "type": "integer" + }, + "path": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "success": { + "type": "boolean" + } + } + } +} +' + +put_es "client_registry_protect_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_uuid": { + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "timestamp": { + "type": "long" + }, + "version": { + "type": "keyword" + }, + "process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "username": { + "type": "keyword" + }, + "action": { + "type": "integer" + }, + "path": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "success": { + "type": "boolean" + } + } + } +} +' + +put_es "gateway_flow_log" ' +{ + "mappings" : { + "properties" : { + "tenant_id": { + "type": "short" + }, + "gateway_id": { + "type": "integer" + }, + "timestamp": { + "type": "long" + }, + "upstream_bytes": { + "type": "long" + }, + "downstream_bytes": { + "type": "long" + } + } + } +} +' + +put_es "mobile_network_log" ' +{ + "settings": { + "index": { + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "keyword" + }, + "device_uuid": { + "type": "keyword" + }, + "local_ip": { + "type": "ip" + }, + "app_name": { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "address": { + "type": "keyword" + }, + "port": { + "type": "short" + }, + "reason": { + "type": "keyword" + } + } + } +} +' + +put_es "mobile_data_outgoing_log" ' +{ + "settings": { + "index": { + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "user_id": { + "type": "long" + }, + "user_name": { + "type": "keyword" + }, + "device_uuid": { + "type": "keyword" + }, + "local_ip": { + "type": "ip" + }, + "app_name": { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "result": { + "type": "byte" + }, + "data_type": { + "type": "byte" + }, + "file_name": { + "type": "keyword", + "index": false + }, + "text_content": { + "type": "keyword", + "index": false + }, + "image_hash": { + "type": "keyword", + "index": false + } + } + } +} +' + +put_es "_settings" ' +{"index.merge.scheduler.max_thread_count" : 1} +' + +put_es "_settings" ' +{ + "index": { + "number_of_replicas": "0" + } +} +' + +put_es "client_ram_protect_log" ' +{ + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_uuid": { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "version": { + "type": "keyword" + }, + "process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "pid": { + "type": "integer" + }, + "user_id": { + "type": "integer" + }, + "user_name": { + "type": "keyword" + }, + "domain_name": { + "type": "keyword" + }, + "action": { + "type": "integer" + } + } + } +}' + +put_es "user_messages" ' +{ + "settings": { + "index": { + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "uid": { + "type": "long" + }, + "message_type": { + "type": "short" + }, + "message_detail_id": { + "type": "long" + }, + "create_time": { + "type": "long" + } + } + } +}' + +put_es "app_store" ' +{ + "settings": { + "analysis": { + "analyzer": { + "ik_smart_pinyin": { + "type": "custom", + "tokenizer": "ik_smart", + "filter": [ + "app_store_pinyin", + "word_delimiter" + ] + }, + "ik_max_word_pinyin": { + "type": "custom", + "tokenizer": "ik_max_word", + "filter": [ + "app_store_pinyin", + "word_delimiter" + ] + } + }, + "filter": { + "app_store_pinyin": { + "type": "pinyin", + "keep_separate_first_letter": true, + "keep_full_pinyin": true, + "keep_original": true, + "limit_first_letter_length": 16, + "lowercase": true, + "remove_duplicated_term": true + } + } + } + }, + "mappings": { + "properties": { + "name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "ik_smart_pinyin" + }, + "desc": { + "type": "text", + "analyzer": "ik_max_word_pinyin", + "search_analyzer": "ik_smart_pinyin" + }, + "version": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "ik_smart_pinyin" + }, + "upload_time": { + "type": "integer" + }, + "change_time": { + "type": "integer" + }, + "path": { + "type": "keyword" + }, + "size": { + "type": "integer" + }, + "file_id": { + "type": "keyword" + }, + "app_type_id": { + "type": "keyword" + }, + "sha256": { + "type": "keyword" + }, + "repo_id": { + "type": "keyword" + }, + "platform": { + "type": "integer" + }, + "adder_id": { + "type": "integer" + } + } + } +}' + + +put_es "app_type" ' +{ + "settings": { + "analysis": { + "analyzer": { + "ik_smart_pinyin": { + "type": "custom", + "tokenizer": "ik_smart", + "filter": [ + "app_store_pinyin", + "word_delimiter" + ] + } + }, + "filter": { + "app_store_pinyin": { + "type": "pinyin", + "keep_separate_first_letter": true, + "keep_full_pinyin": true, + "keep_original": true, + "limit_first_letter_length": 16, + "lowercase": true, + "remove_duplicated_term": true + } + } + } + }, + "mappings": { + "properties": { + "name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "ik_smart_pinyin" + }, + "create_time": { + "type": "integer" + }, + "change_time": { + "type": "integer" + }, + "adder_id": { + "type": "integer" + } + } + } +}' + + +put_es "import_file_log" ' +{ + "settings": { + "index": { + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "tenant_id": { + "type": "short" + }, + "device_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "device_uuid": { + "type": "keyword" + }, + "domain_id": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user_id": { + "type": "long" + }, + "ip": { + "type": "keyword" + }, + "file_name": { + "type": "keyword" + }, + "is_folder": { + "type": "boolean" + }, + "file_size": { + "type": "long" + }, + "file_sha256": { + "type": "keyword" + }, + "file_count": { + "type": "integer" + }, + "action": { + "type": "byte" + }, + "success": { + "type": "boolean" + }, + "approval_id": { + "type": "long" + } + } + } +} +' + +put_es "osi_log" ' +{ + "mappings": { + "properties": { + "device_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "access_url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "origin_url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "user_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "device_uuid": { + "type": "keyword" + }, + "domain_id": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "user_id": { + "type": "long" + }, + "result": { + "type": "boolean" + }, + "interception_reason": { + "type": "short" + } + } + } +} +' + +put_es "sdp_log" ' +{ + "mappings": { + "properties": { + "user_id": { + "type": "long" + }, + "origin_ip": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "timestamp": { + "type": "long" + }, + "access_url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "access_port": { + "type": "long" + }, + "access_result": { + "type": "long" + }, + "device_uuid": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "gateway_id": { + "type": "long" + } + } + } +} +' + +put_es "domain_outside_network_log" ' +{ + "mappings": { + "properties": { + "user_id": { + "type": "long" + }, + "user_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ip": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "timestamp": { + "type": "long" + }, + "access_url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "access_port": { + "type": "long" + }, + "access_result": { + "type": "long" + }, + "device_uuid": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "process_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } +} +' + +put_es "users" ' +{ + "settings": { + "analysis": { + "analyzer": { + "ik_smart_pinyin": { + "type": "custom", + "tokenizer": "name_ngram", + "filter": [ + "user_search_pinyin" + ] + } + }, + "tokenizer": { + "name_ngram": { + "type": "ngram", + "min_gram": 1, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "filter": { + "user_search_pinyin": { + "type": "pinyin", + "keep_separate_first_letter": false, + "keep_joined_full_pinyin": true, + "keep_full_pinyin": true, + "keep_original": true, + "limit_first_letter_length": 16, + "lowercase": true, + "remove_duplicated_term": true + } + } + }, + "index": { + "max_result_window": 114514, + "max_ngram_diff": 20 + } + }, + "mappings": { + "properties": { + "id": { + "type": "integer" + }, + "user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "mobile": { + "type": "keyword" + }, + "avatar": { + "type": "keyword" + }, + "active": { + "type": "integer" + }, + "local_weakly_department_id": { + "type": "long" + }, + "local_strongly_department_id": { + "type": "long" + }, + "ad_id": { + "type": "integer" + }, + "ad_user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ad_display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ad_title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ad_email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ad_mobile": { + "type": "keyword" + }, + "ad_avatar": { + "type": "keyword" + }, + "ad_weakly_department_id": { + "type": "long" + }, + "ad_strongly_department_id": { + "type": "long" + }, + "ldap_id": { + "type": "integer" + }, + "ldap_user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ldap_display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ldap_title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ldap_email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ldap_mobile": { + "type": "keyword" + }, + "ldap_avatar": { + "type": "keyword" + }, + "ldap_weakly_department_id": { + "type": "long" + }, + "ldap_strongly_department_id": { + "type": "long" + }, + "ding_id": { + "type": "integer" + }, + "ding_user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ding_display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ding_title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ding_email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "ding_mobile": { + "type": "keyword" + }, + "ding_avatar": { + "type": "keyword" + }, + "ding_weakly_department_id": { + "type": "long" + }, + "ding_strongly_department_id": { + "type": "long" + }, + "feishu_id": { + "type": "integer" + }, + "feishu_user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "feishu_display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "feishu_title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "feishu_email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "feishu_mobile": { + "type": "keyword" + }, + "feishu_avatar": { + "type": "keyword" + }, + "feishu_weakly_department_id": { + "type": "long" + }, + "feishu_strongly_department_id": { + "type": "long" + }, + "work_weixin_id": { + "type": "integer" + }, + "work_weixin_user_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "work_weixin_display_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "work_weixin_title": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "work_weixin_email": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "work_weixin_mobile": { + "type": "keyword" + }, + "work_weixin_avatar": { + "type": "keyword" + }, + "work_weixin_weakly_department_id": { + "type": "long" + }, + "work_weixin_strongly_department_id": { + "type": "long" + }, + "role": { + "type": "integer" + }, + "weakly_department_id": { + "type": "long" + }, + "strongly_department_id": { + "type": "long" + }, + "department_name": { + "type": "text", + "analyzer": "ik_smart_pinyin", + "search_analyzer": "keyword", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "sso_type": { + "type": "integer" + }, + "path": { + "type": "keyword" + }, + "create_time": { + "type": "integer" + }, + "update_time": { + "type": "integer" + } + } + } +}' + +put_es "watermark_task_log" ' +{ + "settings": { + "index": { + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "id": { + "type": "integer" + }, + "timestamp": { + "type": "long" + }, + "sponsor": { + "type": "keyword", + "fields": { + "wildcard": { + "type": "wildcard" + } + } + }, + "file_name": { + "type": "keyword" + }, + "file_size": { + "type": "long" + }, + "file_md5": { + "type": "keyword" + }, + "action": { + "type": "byte" + }, + "success": { + "type": "boolean" + } + } + } +} +' diff --git a/pkg/resource/es.yaml b/pkg/resource/es.yaml new file mode 100644 index 0000000..ef4a465 --- /dev/null +++ b/pkg/resource/es.yaml @@ -0,0 +1,121 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: db-es +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch + namespace: db-es +spec: + serviceName: elasticsearch + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + volumes: + - name: shared-data + emptyDir: {} + securityContext: + fsGroup: 1000 + initContainers: + - name: fix-permissions + image: hub.yizhisec.com/hybridscope/v2/es-init-helper:alpine-3.22.2 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - | + #/bin/sh + cp -rf /data/plugins/* /app/shared/ + chown -R 1000:1000 /usr/share/elasticsearch/data + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + - name: shared-data + mountPath: /app/shared + securityContext: + runAsUser: 0 + containers: + - name: elasticsearch + image: hub.yizhisec.com/external/elasticsearch:7.17.28 + imagePullPolicy: IfNotPresent + env: + - name: discovery.type + value: single-node + - name: ES_JAVA_OPTS + value: "-Xms%dg -Xmx%dg" + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + - name: shared-data + mountPath: /usr/share/elasticsearch/plugins + resources: + requests: + memory: "%dGi" + cpu: "%d" + limits: + memory: "%dGi" + cpu: "%d" + volumeClaimTemplates: + - metadata: + name: es-data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: longhorn + resources: + requests: + storage: %dGi +--- +apiVersion: v1 +kind: Service +metadata: + name: es-service + namespace: db-es +spec: + type: ClusterIP + selector: + app: elasticsearch + ports: + - name: http + protocol: TCP + port: 9200 + targetPort: http + - name: transport + protocol: TCP + port: 9300 + targetPort: transport +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: es-init-job + namespace: db-es +spec: + template: + spec: + containers: + - name: es-init + image: hub.yizhisec.com/hybridscope/v2/es-init-helper:alpine-3.22.2 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - /data/create_index.sh + restartPolicy: Never + backoffLimit: 2 \ No newline at end of file diff --git a/pkg/resource/flannel.yaml b/pkg/resource/flannel.yaml new file mode 100644 index 0000000..d69fb3f --- /dev/null +++ b/pkg/resource/flannel.yaml @@ -0,0 +1,214 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + k8s-app: flannel + pod-security.kubernetes.io/enforce: privileged + name: kube-flannel +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: flannel + name: flannel + namespace: kube-flannel +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: flannel + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: flannel + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: + - kind: ServiceAccount + name: flannel + namespace: kube-flannel +--- +apiVersion: v1 +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "10.244.0.0/16", + "EnableNFTables": false, + "Backend": { + "Type": "vxlan" + } + } +kind: ConfigMap +metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + name: kube-flannel-cfg + namespace: kube-flannel +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + name: kube-flannel-ds + namespace: kube-flannel +spec: + selector: + matchLabels: + app: flannel + k8s-app: flannel + template: + metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --ip-masq + - --kube-subnet-mgr + command: + - /opt/bin/flanneld + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EVENT_QUEUE_DEPTH + value: "5000" + - name: CONT_WHEN_CACHE_NOT_READY + value: "false" + image: ghcr.io/flannel-io/flannel:v0.27.4 + name: kube-flannel + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + privileged: false + volumeMounts: + - mountPath: /run/flannel + name: run + - mountPath: /etc/kube-flannel/ + name: flannel-cfg + - mountPath: /run/xtables.lock + name: xtables-lock + hostNetwork: true + initContainers: + - args: + - -f + - /flannel + - /opt/cni/bin/flannel + command: + - cp + image: ghcr.io/flannel-io/flannel-cni-plugin:v1.8.0-flannel1 + name: install-cni-plugin + volumeMounts: + - mountPath: /opt/cni/bin + name: cni-plugin + - args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + command: + - cp + image: ghcr.io/flannel-io/flannel:v0.27.4 + name: install-cni + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni + - mountPath: /etc/kube-flannel/ + name: flannel-cfg + priorityClassName: system-node-critical + serviceAccountName: flannel + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /run/flannel + name: run + - hostPath: + path: /opt/cni/bin + name: cni-plugin + - hostPath: + path: /etc/cni/net.d + name: cni + - configMap: + name: kube-flannel-cfg + name: flannel-cfg + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock \ No newline at end of file diff --git a/pkg/resource/kibana.yaml b/pkg/resource/kibana.yaml new file mode 100644 index 0000000..a443d41 --- /dev/null +++ b/pkg/resource/kibana.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + namespace: db-es +spec: + replicas: 0 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: hub.yizhisec.com/external/kibana:7.17.28 + imagePullPolicy: IfNotPresent + env: + - name: ELASTICSEARCH_HOSTS + value: http://es-service:9200 + - name: SERVER_HOST + value: 0.0.0.0 + ports: + - containerPort: 5601 + name: http + resources: + limits: + memory: 2Gi + cpu: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana-service + namespace: db-es +spec: + type: NodePort + selector: + app: kibana + ports: + - name: http + protocol: TCP + port: 5601 + targetPort: 5601 + nodePort: 31601 diff --git a/pkg/resource/less-dns.yaml b/pkg/resource/less-dns.yaml new file mode 100644 index 0000000..7fa82d1 --- /dev/null +++ b/pkg/resource/less-dns.yaml @@ -0,0 +1,83 @@ +# k8s-hs-less-dns.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hs-net +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-less-dns + namespace: hs-net +data: + config.yml: | + { + "log": { + "level": "info" + }, + "vnet4": "100.64.0.1/10", + "vnet6": "fc00:eeaa:0000:0000::/48", + "redis": { + "custom": [ + { + "username": null, + "password": "HybridScope0xRed1s.", + "host": "redis-master.db-redis", + "port": 6379, + "tls_insecure": null, + "db": 10 + } + ] + }, + "mqtt": { + "client_id": "dns_mqtt_client", + "protocol": "tls", + "host": "emqx-service.db-emqx", + "port": 1883, + "cert": "", + "key": "", + "keep_alive": 60 + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: less-dns + namespace: hs-net + labels: + app: less-dns +spec: + replicas: 1 + selector: + matchLabels: + app: less-dns + template: + metadata: + labels: + app: less-dns + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: less-dns + containers: + - name: less-dns + image: hub.yizhisec.com/hybridscope/less_dns_service:latest + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config-volume + mountPath: /etc/less_dns_service + securityContext: + privileged: true + volumes: + - name: config-volume + configMap: + name: config-less-dns + items: + - key: config.yml + path: config.yml + restartPolicy: Always diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go new file mode 100644 index 0000000..0506164 --- /dev/null +++ b/pkg/resource/resource.go @@ -0,0 +1,28 @@ +package resource + +import ( + _ "embed" +) + +var ( + //go:embed flannel.yaml + YAMLFlannel []byte + + //go:embed es.yaml + YAMLES string + + //go:embed kibana.yaml + YAMLKibana []byte + + //go:embed es.init.sh + BashESInit []byte + + //go:embed emqx.yaml + YAMLEMQX []byte + + //go:embed yosguard.create.sql + SQLYosguard []byte + + //go:embed less-dns.yaml + YAMLLessDNS []byte +) diff --git a/pkg/resource/yosguard.create.sql b/pkg/resource/yosguard.create.sql new file mode 100644 index 0000000..0a23b07 --- /dev/null +++ b/pkg/resource/yosguard.create.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS `pkg` +( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `path` TEXT, + `installed` INTEGER, + `create_timestamp` INTEGER, + `install_timestamp` INTEGER +); + +CREATE TABLE IF NOT EXISTS `patch` +( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `path` TEXT, + `verified` INTEGER, + `installed` INTEGER, + `create_timestamp` INTEGER, + `verify_timestamp` INTEGER, + `install_timestamp` INTEGER +); + +-- 记录注册的机器的信息 +CREATE TABLE IF NOT EXISTS `machine` +( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `uuid` CHAR(32), + `create_timestamp` INTEGER +); + +-- 记录下发的指令 +CREATE TABLE IF NOT EXISTS `action` +( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `uuid` CHAR(32), + `action` INTEGER, -- 要下发的指令 + `create_timestamp` INTEGER -- 下发命令的时间戳，秒为单位 +); \ No newline at end of file