feat: add registry config, image upload/download, and OCI format support
Backend: - Add registry_address configuration API (GET/POST) - Add tar image upload with OCI and Docker format support - Add image download with streaming optimization - Fix blob download using c.Send (Fiber v3 SendStream bug) - Add registry_address prefix stripping for all OCI v2 endpoints - Add AGENTS.md for project documentation Frontend: - Add settings store with Snackbar notifications - Add image upload dialog with progress bar - Add download state tracking with multi-stage feedback - Replace alert() with MUI Snackbar messages - Display image names without registry_address prefix 🤖 Generated with [Qoder](https://qoder.com)
This commit is contained in:
loveuer
416
internal/module/registry/handler.upload.go
Normal file
416
internal/module/registry/handler.upload.go
Normal file
@@ -0,0 +1,416 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.loveuer.com/loveuer/cluster/internal/model"
|
||||
"gitea.loveuer.com/loveuer/cluster/pkg/resp"
|
||||
"gitea.loveuer.com/loveuer/cluster/pkg/store"
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type DockerManifestItem struct {
|
||||
Config string `json:"Config"`
|
||||
RepoTags []string `json:"RepoTags"`
|
||||
Layers []string `json:"Layers"`
|
||||
}
|
||||
|
||||
type OCIIndex struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Manifests []OCIManifestDescriptor `json:"manifests"`
|
||||
}
|
||||
|
||||
type OCIManifestDescriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
Annotations map[string]string `json:"annotations,omitempty"`
|
||||
}
|
||||
|
||||
type OCIManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType,omitempty"`
|
||||
Config OCIDescriptor `json:"config"`
|
||||
Layers []OCIDescriptor `json:"layers"`
|
||||
}
|
||||
|
||||
type OCIDescriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func RegistryImageUpload(ctx context.Context, db *gorm.DB, store store.Store) fiber.Handler {
|
||||
return func(c fiber.Ctx) error {
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
return resp.R400(c, "MISSING_FILE", nil, "file is required")
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(file.Filename, ".tar") {
|
||||
return resp.R400(c, "INVALID_FILE_TYPE", nil, "only .tar files are allowed")
|
||||
}
|
||||
|
||||
fileReader, err := file.Open()
|
||||
if err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to open file: %w", err))
|
||||
}
|
||||
defer fileReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(fileReader)
|
||||
|
||||
var manifestItems []DockerManifestItem
|
||||
var ociIndex *OCIIndex
|
||||
blobContents := make(map[string][]byte)
|
||||
layerContents := make(map[string][]byte)
|
||||
var configContent []byte
|
||||
var configDigest string
|
||||
var indexJSON []byte
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to read tar: %w", err))
|
||||
}
|
||||
|
||||
content, err := io.ReadAll(tarReader)
|
||||
if err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to read file content: %w", err))
|
||||
}
|
||||
|
||||
switch header.Name {
|
||||
case "manifest.json":
|
||||
if err := json.Unmarshal(content, &manifestItems); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to parse manifest.json: %w", err))
|
||||
}
|
||||
case "index.json":
|
||||
indexJSON = content
|
||||
if err := json.Unmarshal(content, &ociIndex); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to parse index.json: %w", err))
|
||||
}
|
||||
default:
|
||||
if strings.HasSuffix(header.Name, ".json") && header.Name != "manifest.json" && header.Name != "index.json" {
|
||||
configContent = content
|
||||
hash := sha256.Sum256(content)
|
||||
configDigest = "sha256:" + hex.EncodeToString(hash[:])
|
||||
} else if strings.HasSuffix(header.Name, "/layer") {
|
||||
layerContents[header.Name] = content
|
||||
} else if strings.Contains(header.Name, "blobs/sha256/") && !strings.HasSuffix(header.Name, "/") {
|
||||
blobContents[header.Name] = content
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle OCI format
|
||||
if ociIndex != nil && len(ociIndex.Manifests) > 0 {
|
||||
return handleOCIFormat(c, db, store, ociIndex, blobContents, indexJSON)
|
||||
}
|
||||
|
||||
// Handle Docker format
|
||||
if len(manifestItems) == 0 {
|
||||
return resp.R400(c, "INVALID_TAR", nil, "manifest.json or index.json not found in tar file")
|
||||
}
|
||||
|
||||
manifestItem := manifestItems[0]
|
||||
if len(manifestItem.RepoTags) == 0 {
|
||||
return resp.R400(c, "INVALID_MANIFEST", nil, "no RepoTags found in manifest")
|
||||
}
|
||||
|
||||
// Extract original repository and tag from tar file
|
||||
// e.g., "docker.io/redis:latest" -> repo: "docker.io/redis", tag: "latest"
|
||||
originalRepoTag := manifestItem.RepoTags[0]
|
||||
parts := strings.SplitN(originalRepoTag, ":", 2)
|
||||
originalRepo := parts[0]
|
||||
tag := "latest"
|
||||
if len(parts) == 2 {
|
||||
tag = parts[1]
|
||||
}
|
||||
|
||||
// Store only the original repository name (without registry_address prefix)
|
||||
// This allows registry_address to be changed without breaking existing images
|
||||
repoName := originalRepo
|
||||
|
||||
if err := db.FirstOrCreate(&model.Repository{}, model.Repository{Name: repoName}).Error; err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to create repository: %w", err))
|
||||
}
|
||||
|
||||
if err := store.CreatePartition(c.Context(), "registry"); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to create partition: %w", err))
|
||||
}
|
||||
|
||||
if configContent != nil {
|
||||
if err := store.WriteBlob(c.Context(), configDigest, strings.NewReader(string(configContent))); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write config blob: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Blob{
|
||||
Digest: configDigest,
|
||||
Size: int64(len(configContent)),
|
||||
MediaType: "application/vnd.docker.container.image.v1+json",
|
||||
Repository: repoName,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save config blob metadata: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
layerDigests := make([]map[string]interface{}, 0, len(manifestItem.Layers))
|
||||
for _, layerPath := range manifestItem.Layers {
|
||||
content, ok := layerContents[layerPath]
|
||||
if !ok {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("layer %s not found in tar", layerPath))
|
||||
}
|
||||
|
||||
hash := sha256.Sum256(content)
|
||||
digest := "sha256:" + hex.EncodeToString(hash[:])
|
||||
|
||||
if err := store.WriteBlob(c.Context(), digest, strings.NewReader(string(content))); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write layer blob: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Blob{
|
||||
Digest: digest,
|
||||
Size: int64(len(content)),
|
||||
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
Repository: repoName,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save layer blob metadata: %w", err))
|
||||
}
|
||||
|
||||
layerDigests = append(layerDigests, map[string]interface{}{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": len(content),
|
||||
"digest": digest,
|
||||
})
|
||||
}
|
||||
|
||||
manifestData := map[string]interface{}{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": map[string]interface{}{
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": len(configContent),
|
||||
"digest": configDigest,
|
||||
},
|
||||
"layers": layerDigests,
|
||||
}
|
||||
|
||||
manifestJSON, err := json.Marshal(manifestData)
|
||||
if err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to marshal manifest: %w", err))
|
||||
}
|
||||
|
||||
manifestHash := sha256.Sum256(manifestJSON)
|
||||
manifestDigest := "sha256:" + hex.EncodeToString(manifestHash[:])
|
||||
|
||||
if err := store.WriteManifest(c.Context(), manifestDigest, manifestJSON); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write manifest: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Manifest{
|
||||
Repository: repoName,
|
||||
Tag: tag,
|
||||
Digest: manifestDigest,
|
||||
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
|
||||
Size: int64(len(manifestJSON)),
|
||||
Content: manifestJSON,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save manifest: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Tag{
|
||||
Repository: repoName,
|
||||
Tag: tag,
|
||||
Digest: manifestDigest,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save tag: %w", err))
|
||||
}
|
||||
|
||||
return resp.R200(c, map[string]interface{}{
|
||||
"message": "upload success",
|
||||
"repository": repoName,
|
||||
"tag": tag,
|
||||
"digest": manifestDigest,
|
||||
"original_tag": originalRepoTag,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func handleOCIFormat(c fiber.Ctx, db *gorm.DB, store store.Store, ociIndex *OCIIndex, blobContents map[string][]byte, indexJSON []byte) error {
|
||||
// Get image name and tag from index annotations
|
||||
if len(ociIndex.Manifests) == 0 {
|
||||
return resp.R400(c, "INVALID_OCI_INDEX", nil, "no manifests found in index.json")
|
||||
}
|
||||
|
||||
manifestDesc := ociIndex.Manifests[0]
|
||||
imageName := ""
|
||||
tag := "latest"
|
||||
|
||||
if manifestDesc.Annotations != nil {
|
||||
if name, ok := manifestDesc.Annotations["io.containerd.image.name"]; ok {
|
||||
imageName = name
|
||||
} else if name, ok := manifestDesc.Annotations["org.opencontainers.image.ref.name"]; ok {
|
||||
tag = name
|
||||
}
|
||||
}
|
||||
|
||||
if imageName == "" {
|
||||
return resp.R400(c, "INVALID_OCI_INDEX", nil, "image name not found in annotations")
|
||||
}
|
||||
|
||||
// Extract repo and tag from image name
|
||||
parts := strings.SplitN(imageName, ":", 2)
|
||||
repoName := parts[0]
|
||||
if len(parts) == 2 {
|
||||
tag = parts[1]
|
||||
}
|
||||
|
||||
// Create repository
|
||||
if err := db.FirstOrCreate(&model.Repository{}, model.Repository{Name: repoName}).Error; err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to create repository: %w", err))
|
||||
}
|
||||
|
||||
if err := store.CreatePartition(c.Context(), "registry"); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to create partition: %w", err))
|
||||
}
|
||||
|
||||
// Store index.json as a blob
|
||||
indexHash := sha256.Sum256(indexJSON)
|
||||
indexDigest := "sha256:" + hex.EncodeToString(indexHash[:])
|
||||
if err := store.WriteBlob(c.Context(), indexDigest, strings.NewReader(string(indexJSON))); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write index blob: %w", err))
|
||||
}
|
||||
if err := db.Create(&model.Blob{
|
||||
Digest: indexDigest,
|
||||
Size: int64(len(indexJSON)),
|
||||
MediaType: "application/vnd.oci.image.index.v1+json",
|
||||
Repository: repoName,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save index blob metadata: %w", err))
|
||||
}
|
||||
|
||||
// Process the manifest blob
|
||||
manifestBlobPath := "blobs/sha256/" + strings.TrimPrefix(manifestDesc.Digest, "sha256:")
|
||||
manifestContent, ok := blobContents[manifestBlobPath]
|
||||
if !ok {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("manifest blob %s not found in tar", manifestBlobPath))
|
||||
}
|
||||
|
||||
// Parse OCI manifest
|
||||
var ociManifest OCIManifest
|
||||
if err := json.Unmarshal(manifestContent, &ociManifest); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to parse OCI manifest: %w", err))
|
||||
}
|
||||
|
||||
// Store config blob
|
||||
configBlobPath := "blobs/sha256/" + strings.TrimPrefix(ociManifest.Config.Digest, "sha256:")
|
||||
configContent, ok := blobContents[configBlobPath]
|
||||
if !ok {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("config blob %s not found in tar", configBlobPath))
|
||||
}
|
||||
|
||||
if err := store.WriteBlob(c.Context(), ociManifest.Config.Digest, strings.NewReader(string(configContent))); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write config blob: %w", err))
|
||||
}
|
||||
if err := db.Create(&model.Blob{
|
||||
Digest: ociManifest.Config.Digest,
|
||||
Size: ociManifest.Config.Size,
|
||||
MediaType: ociManifest.Config.MediaType,
|
||||
Repository: repoName,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save config blob metadata: %w", err))
|
||||
}
|
||||
|
||||
// Store layer blobs
|
||||
for _, layer := range ociManifest.Layers {
|
||||
layerBlobPath := "blobs/sha256/" + strings.TrimPrefix(layer.Digest, "sha256:")
|
||||
layerContent, ok := blobContents[layerBlobPath]
|
||||
if !ok {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("layer blob %s not found in tar", layerBlobPath))
|
||||
}
|
||||
|
||||
if err := store.WriteBlob(c.Context(), layer.Digest, strings.NewReader(string(layerContent))); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write layer blob: %w", err))
|
||||
}
|
||||
if err := db.Create(&model.Blob{
|
||||
Digest: layer.Digest,
|
||||
Size: layer.Size,
|
||||
MediaType: layer.MediaType,
|
||||
Repository: repoName,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save layer blob metadata: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
// Convert OCI manifest to Docker manifest v2 format for compatibility
|
||||
manifestData := map[string]interface{}{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": map[string]interface{}{
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": ociManifest.Config.Size,
|
||||
"digest": ociManifest.Config.Digest,
|
||||
},
|
||||
"layers": []map[string]interface{}{},
|
||||
}
|
||||
|
||||
layers := []map[string]interface{}{}
|
||||
for _, layer := range ociManifest.Layers {
|
||||
layers = append(layers, map[string]interface{}{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": layer.Size,
|
||||
"digest": layer.Digest,
|
||||
})
|
||||
}
|
||||
manifestData["layers"] = layers
|
||||
|
||||
manifestJSON, err := json.Marshal(manifestData)
|
||||
if err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to marshal manifest: %w", err))
|
||||
}
|
||||
|
||||
manifestHash := sha256.Sum256(manifestJSON)
|
||||
manifestDigest := "sha256:" + hex.EncodeToString(manifestHash[:])
|
||||
|
||||
if err := store.WriteManifest(c.Context(), manifestDigest, manifestJSON); err != nil {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to write manifest: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Manifest{
|
||||
Repository: repoName,
|
||||
Tag: tag,
|
||||
Digest: manifestDigest,
|
||||
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
|
||||
Size: int64(len(manifestJSON)),
|
||||
Content: manifestJSON,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save manifest: %w", err))
|
||||
}
|
||||
|
||||
if err := db.Create(&model.Tag{
|
||||
Repository: repoName,
|
||||
Tag: tag,
|
||||
Digest: manifestDigest,
|
||||
}).Error; err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||
return resp.R500(c, "", nil, fmt.Errorf("failed to save tag: %w", err))
|
||||
}
|
||||
|
||||
return resp.R200(c, map[string]interface{}{
|
||||
"message": "upload success (OCI format)",
|
||||
"repository": repoName,
|
||||
"tag": tag,
|
||||
"digest": manifestDigest,
|
||||
"original_tag": imageName,
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user