feat: mem, local uploader
This commit is contained in:
loveuer
234
internal/interfaces/manifests/db.go
Normal file
234
internal/interfaces/manifests/db.go
Normal file
@ -0,0 +1,234 @@
|
||||
package manifests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/samber/lo"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gorm.io/gorm"
|
||||
"io"
|
||||
"net/http"
|
||||
"nf-repo/internal/interfaces"
|
||||
"nf-repo/internal/model"
|
||||
"nf-repo/internal/util/rerr"
|
||||
"nf-repo/internal/util/tools"
|
||||
)
|
||||
|
||||
type PackageManifest struct {
|
||||
Id uint64 `json:"id" gorm:"primaryKey;column:id"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"column:created_at;autoCreateTime:milli"`
|
||||
UpdatedAt int64 `json:"updated_at" gorm:"column:updated_at;autoUpdateTime:milli"`
|
||||
|
||||
Repo string `json:"repo" gorm:"uniqueIndex:repo_tag_idx;column:repo"`
|
||||
Target string `json:"target" gorm:"uniqueIndex:repo_tag_idx;column:target"`
|
||||
Digest string `json:"digest" gorm:"unique;column:digest"`
|
||||
ContentType string `json:"content_type" gorm:"column:content_type"`
|
||||
Content []byte `json:"content" gorm:"column:content;type:bytes"`
|
||||
}
|
||||
|
||||
type dbManifests struct {
|
||||
db interfaces.Database
|
||||
}
|
||||
|
||||
func (m *dbManifests) Get(ctx context.Context, repo string, target string) (io.ReadCloser, string, *rerr.RepositoryError) {
|
||||
var (
|
||||
err error
|
||||
pm = new(PackageManifest)
|
||||
h model.Hash
|
||||
tx = m.db.TX(tools.Timeout(5)).Model(pm)
|
||||
)
|
||||
|
||||
if h, err = model.NewHash(target); err == nil {
|
||||
tx = tx.Where("digest", h.String())
|
||||
} else {
|
||||
tx = tx.Where("repo", repo).
|
||||
Where("target", target)
|
||||
}
|
||||
|
||||
if err = tx.
|
||||
Take(pm).
|
||||
Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, "", &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: fmt.Sprintf("Unknown name: %s@%s", repo, target),
|
||||
}
|
||||
}
|
||||
|
||||
return nil, "", rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(pm.Content)), pm.ContentType, nil
|
||||
}
|
||||
|
||||
func (m *dbManifests) Put(ctx context.Context, repo string, target string, digest string, mf *model.Manifest) *rerr.RepositoryError {
|
||||
var (
|
||||
err error
|
||||
pm = &PackageManifest{
|
||||
Repo: repo,
|
||||
Target: target,
|
||||
Digest: digest,
|
||||
ContentType: mf.ContentType,
|
||||
Content: mf.Blob,
|
||||
}
|
||||
)
|
||||
|
||||
// todo on conflict
|
||||
if err = m.db.TX(tools.Timeout(5)).Create(pm).Error; err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
logrus.
|
||||
WithField("path", "dbManifests.Put.Create").
|
||||
WithField("err", err.Error()).
|
||||
Trace()
|
||||
|
||||
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
|
||||
Where("(repo = ? AND target = ?) OR (digest = ?)", repo, target, digest).
|
||||
Updates(map[string]any{
|
||||
"repo": repo,
|
||||
"target": target,
|
||||
"digest": digest,
|
||||
"content_type": mf.ContentType,
|
||||
"content": mf.Blob,
|
||||
}).
|
||||
Error; err != nil {
|
||||
logrus.
|
||||
WithField("path", "dbManifests.Put.Updates").
|
||||
WithField("err", err.Error()).
|
||||
Debug()
|
||||
return rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *dbManifests) Delete(ctx context.Context, repo string, target string) *rerr.RepositoryError {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
if err = m.db.TX(tools.Timeout(5)).
|
||||
Where("repo", repo).
|
||||
Where("target", target).
|
||||
Delete(&PackageManifest{}).
|
||||
Error; err != nil {
|
||||
return rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *dbManifests) Catelog(ctx context.Context, limit int, last int) (*model.Catalog, *rerr.RepositoryError) {
|
||||
var (
|
||||
err error
|
||||
list = make([]*PackageManifest, 0)
|
||||
)
|
||||
|
||||
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
|
||||
Order("updated_at").
|
||||
Offset(last).
|
||||
Limit(limit).
|
||||
Find(&list).
|
||||
Error; err != nil {
|
||||
return nil, rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return &model.Catalog{
|
||||
Repos: lo.Map(list, func(item *PackageManifest, index int) string {
|
||||
return item.Repo
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *dbManifests) Tags(ctx context.Context, repo string, limit, last int) (*model.Tag, *rerr.RepositoryError) {
|
||||
var (
|
||||
err error
|
||||
list = make([]*PackageManifest, 0)
|
||||
)
|
||||
|
||||
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
|
||||
Where("repo", repo).
|
||||
Order("updated_at").
|
||||
Offset(last).
|
||||
Limit(limit).
|
||||
Find(&list).
|
||||
Error; err != nil {
|
||||
return nil, rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return &model.Tag{
|
||||
Name: repo,
|
||||
Tags: lo.Map(list, func(item *PackageManifest, index int) string {
|
||||
return item.Target
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *dbManifests) Referrers(ctx context.Context, repo string, target string) (*model.IndexManifest, *rerr.RepositoryError) {
|
||||
var (
|
||||
err error
|
||||
pm = new(PackageManifest)
|
||||
manifest = &model.IndexManifest{}
|
||||
tx = m.db.TX(tools.Timeout(5)).Model(pm)
|
||||
)
|
||||
|
||||
h, err := model.NewHash(target)
|
||||
if err != nil {
|
||||
tx = tx.Where("repo", repo).Where("digest", h.String())
|
||||
} else {
|
||||
tx = tx.Where("repo", repo).Where("target", target)
|
||||
}
|
||||
|
||||
if err = m.db.TX(tools.Timeout(5)).Model(pm).
|
||||
Take(pm).
|
||||
Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: fmt.Sprintf("Unknown name: %s@%s", repo, target),
|
||||
}
|
||||
}
|
||||
|
||||
logrus.
|
||||
WithField("path", "dbManifests.Referrers.Take").
|
||||
WithField("repo", repo).
|
||||
WithField("target", target).
|
||||
WithField("err", err.Error()).
|
||||
Debug()
|
||||
|
||||
return nil, rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(pm.Content, manifest); err != nil {
|
||||
logrus.
|
||||
WithField("path", "dbManifests.Referrers.Unmarshal").
|
||||
WithField("repo", repo).
|
||||
WithField("target", target).
|
||||
WithField("err", err.Error()).
|
||||
Debug()
|
||||
return nil, rerr.ErrInternal(err)
|
||||
}
|
||||
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func NewManifestDBHandler(tx interfaces.Database) interfaces.ManifestHandler {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
if err = tx.TX(tools.Timeout(5)).AutoMigrate(&PackageManifest{}); err != nil {
|
||||
logrus.
|
||||
WithField("path", "NewManifestDBHandler").
|
||||
WithField("method", "AutoMigrate").
|
||||
Panic(err)
|
||||
}
|
||||
|
||||
return &dbManifests{db: tx}
|
||||
}
|
||||
278
internal/interfaces/manifests/mem.go
Normal file
278
internal/interfaces/manifests/mem.go
Normal file
@ -0,0 +1,278 @@
|
||||
package manifests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/sirupsen/logrus"
|
||||
"io"
|
||||
"net/http"
|
||||
"nf-repo/internal/interfaces"
|
||||
"nf-repo/internal/model"
|
||||
"nf-repo/internal/model/types"
|
||||
"nf-repo/internal/util/rerr"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type memManifest struct {
|
||||
sync.RWMutex
|
||||
m map[string]map[string]*model.Manifest
|
||||
}
|
||||
|
||||
func (m *memManifest) Referrers(ctx context.Context, repo string, target string) (*model.IndexManifest, *rerr.RepositoryError) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
digestToManifestMap, repoExists := m.m[repo]
|
||||
if !repoExists {
|
||||
return nil, &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: "Unknown name",
|
||||
}
|
||||
}
|
||||
|
||||
im := &model.IndexManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: types.OCIImageIndex,
|
||||
Manifests: []model.Descriptor{},
|
||||
}
|
||||
|
||||
for digest, manifest := range digestToManifestMap {
|
||||
h, err := model.NewHash(digest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var refPointer struct {
|
||||
Subject *model.Descriptor `json:"subject"`
|
||||
}
|
||||
|
||||
json.Unmarshal(manifest.Blob, &refPointer)
|
||||
if refPointer.Subject == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
referenceDigest := refPointer.Subject.Digest
|
||||
if referenceDigest.String() != target {
|
||||
continue
|
||||
}
|
||||
// At this point, we know the current digest references the target
|
||||
var imageAsArtifact struct {
|
||||
Config struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
} `json:"config"`
|
||||
}
|
||||
|
||||
json.Unmarshal(manifest.Blob, &imageAsArtifact)
|
||||
|
||||
im.Manifests = append(im.Manifests, model.Descriptor{
|
||||
MediaType: types.MediaType(manifest.ContentType),
|
||||
Size: int64(len(manifest.Blob)),
|
||||
Digest: h,
|
||||
ArtifactType: imageAsArtifact.Config.MediaType,
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
return im, nil
|
||||
}
|
||||
|
||||
func (m *memManifest) Tags(ctx context.Context, repo string, limit int, last int) (*model.Tag, *rerr.RepositoryError) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
c, ok := m.m[repo]
|
||||
if !ok {
|
||||
return nil, &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: "Unknown name",
|
||||
}
|
||||
}
|
||||
|
||||
var tags []string
|
||||
for tag := range c {
|
||||
if !strings.Contains(tag, "sha256:") {
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
sort.Strings(tags)
|
||||
|
||||
// https://github.com/opencontainers/distribution-spec/blob/b505e9cc53ec499edbd9c1be32298388921bb705/detail.md#tags-paginated
|
||||
// Offset using last query parameter.
|
||||
//if last := ctx.Query("last"); last != "" {
|
||||
// for i, t := range tags {
|
||||
// if t > last {
|
||||
// tags = tags[i:]
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//// Limit using n query parameter.
|
||||
//if ns := ctx.Query("n"); ns != "" {
|
||||
// if n, err := strconv.Atoi(ns); err != nil {
|
||||
// return rerr.Error(ctx, &rerr.RepositoryError{
|
||||
// Status: http.StatusBadRequest,
|
||||
// Code: "BAD_REQUEST",
|
||||
// Message: fmt.Sprintf("parsing n: %v", err),
|
||||
// })
|
||||
// } else if n < len(tags) {
|
||||
// tags = tags[:n]
|
||||
// }
|
||||
//}
|
||||
|
||||
tagsToList := &model.Tag{
|
||||
Name: repo,
|
||||
Tags: tags,
|
||||
}
|
||||
|
||||
return tagsToList, nil
|
||||
}
|
||||
|
||||
func (m *memManifest) Catelog(ctx context.Context, limit, last int) (*model.Catalog, *rerr.RepositoryError) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
var repos []string
|
||||
countRepos := 0
|
||||
|
||||
// TODO: implement pagination
|
||||
for key := range m.m {
|
||||
if countRepos >= limit {
|
||||
break
|
||||
}
|
||||
countRepos++
|
||||
|
||||
repos = append(repos, key)
|
||||
}
|
||||
|
||||
repositoriesToList := &model.Catalog{
|
||||
Repos: repos,
|
||||
}
|
||||
|
||||
return repositoriesToList, nil
|
||||
}
|
||||
|
||||
func (m *memManifest) Put(ctx context.Context, repo string, target string, digest string, mf *model.Manifest) *rerr.RepositoryError {
|
||||
// If the manifest
|
||||
// list's constituent manifests are already uploaded.
|
||||
// This isn't strictly required by the registry API, but some
|
||||
// registries require this.
|
||||
if types.MediaType(mf.ContentType).IsIndex() {
|
||||
if err := func() *rerr.RepositoryError {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
im, err := model.ParseIndexManifest(bytes.NewReader(mf.Blob))
|
||||
if err != nil {
|
||||
return &rerr.RepositoryError{
|
||||
Status: http.StatusBadRequest,
|
||||
Code: "MANIFEST_INVALID",
|
||||
Message: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
for _, desc := range im.Manifests {
|
||||
if !desc.MediaType.IsDistributable() {
|
||||
continue
|
||||
}
|
||||
|
||||
if desc.MediaType.IsIndex() || desc.MediaType.IsImage() {
|
||||
if _, found := m.m[repo][desc.Digest.String()]; !found {
|
||||
return &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "MANIFEST_UNKNOWN",
|
||||
Message: fmt.Sprintf("Sub-manifest %q not found", desc.Digest),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// TODO: Probably want to do an existence check for blobs.
|
||||
logrus.Warnf("TODO: Check blobs for %q", desc.Digest)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if _, ok := m.m[repo]; !ok {
|
||||
m.m[repo] = make(map[string]*model.Manifest, 2)
|
||||
}
|
||||
|
||||
// Allow future references by target (tag) and immutable digest.
|
||||
// See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-by-digest-immutable-identifier.
|
||||
m.m[repo][digest] = mf
|
||||
m.m[repo][target] = mf
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memManifest) Delete(ctx context.Context, repo string, target string) *rerr.RepositoryError {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if _, ok := m.m[repo]; !ok {
|
||||
return &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: "Unknown name",
|
||||
}
|
||||
}
|
||||
|
||||
_, ok := m.m[repo][target]
|
||||
if !ok {
|
||||
return &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "MANIFEST_UNKNOWN",
|
||||
Message: "Unknown manifest",
|
||||
}
|
||||
}
|
||||
|
||||
delete(m.m[repo], target)
|
||||
|
||||
if len(m.m[repo]) == 0 {
|
||||
delete(m.m, repo)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memManifest) Get(ctx context.Context, repo string, target string) (io.ReadCloser, string, *rerr.RepositoryError) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
c, ok := m.m[repo]
|
||||
if !ok {
|
||||
return nil, "", &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "NAME_UNKNOWN",
|
||||
Message: "Unknown name",
|
||||
}
|
||||
}
|
||||
|
||||
f, ok := c[target]
|
||||
if !ok {
|
||||
return nil, "", &rerr.RepositoryError{
|
||||
Status: http.StatusNotFound,
|
||||
Code: "MANIFEST_UNKNOWN",
|
||||
Message: "Unknown manifest",
|
||||
}
|
||||
}
|
||||
|
||||
reader := io.NopCloser(bytes.NewReader(f.Blob))
|
||||
|
||||
return reader, f.ContentType, nil
|
||||
}
|
||||
|
||||
func NewManifestMemHandler() interfaces.ManifestHandler {
|
||||
return &memManifest{m: make(map[string]map[string]*model.Manifest)}
|
||||
}
|
||||
Reference in New Issue
Block a user