loveuer/repo.me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: mem, local uploader

Browse Source
This commit is contained in:
loveuer 2024-04-10 22:10:09 +08:00
commit c5d0b8e45b
40 changed files with 8261 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
.idea
.vscode
*.crt
*.key
*.sqlite
*.db
images

5484
api.md Normal file
View File

File diff suppressed because it is too large Load Diff

28
go.mod Normal file
View File

@ -0,0 +1,28 @@
module nf-repo
go 1.20
require (
github.com/glebarez/sqlite v1.11.0
github.com/google/go-containerregistry v0.19.1
github.com/loveuer/nf v0.1.6
github.com/samber/lo v1.39.0
github.com/sirupsen/logrus v1.9.3
gorm.io/gorm v1.25.9
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
golang.org/x/sys v0.15.0 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.23.1 // indirect
)

54
go.sum Normal file
View File

@ -0,0 +1,54 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY=
github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/loveuer/nf v0.1.6 h1:YIhrQm1iIMkTqP6EOdQ7OsNoFXCD0/i7oi3yjaMtpCU=
github.com/loveuer/nf v0.1.6/go.mod h1:uKsKYym27ravyTXSBSnxU86V7osxx9cM6DJ+dVBfJ1Q=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=

BIN
internal/.DS_Store vendored Normal file
View File

Binary file not shown.

31
internal/and/and_closer.go Normal file
View File

@ -0,0 +1,31 @@
package and
import (
"io"
)
type ReadCloser struct {
io.Reader
CloseFunc func() error
}
var _ io.ReadCloser = (*ReadCloser)(nil)
// Close implements io.ReadCloser
func (rac *ReadCloser) Close() error {
return rac.CloseFunc()
}
// WriteCloser implements io.WriteCloser by reading from a particular io.Writer
// and then calling the provided "Close()" method.
type WriteCloser struct {
io.Writer
CloseFunc func() error
}
var _ io.WriteCloser = (*WriteCloser)(nil)
// Close implements io.WriteCloser
func (wac *WriteCloser) Close() error {
return wac.CloseFunc()
}

85
internal/and/and_closer_test.go Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package and
import (
"bytes"
"io"
"testing"
)
func TestRead(t *testing.T) {
want := "asdf"
r := bytes.NewBufferString(want)
called := false
rac := &ReadCloser{
Reader: r,
CloseFunc: func() error {
called = true
return nil
},
}
data, err := io.ReadAll(rac)
if err != nil {
t.Errorf("ReadAll(rac) = %v", err)
}
if got := string(data); got != want {
t.Errorf("ReadAll(rac); got %q, want %q", got, want)
}
if called {
t.Error("called before Close(); got true, wanted false")
}
if err := rac.Close(); err != nil {
t.Errorf("Close() = %v", err)
}
if !called {
t.Error("called after Close(); got false, wanted true")
}
}
func TestWrite(t *testing.T) {
w := bytes.NewBuffer([]byte{})
called := false
wac := &WriteCloser{
Writer: w,
CloseFunc: func() error {
called = true
return nil
},
}
want := "asdf"
if _, err := wac.Write([]byte(want)); err != nil {
t.Errorf("Write(%q); = %v", want, err)
}
if called {
t.Error("called before Close(); got true, wanted false")
}
if err := wac.Close(); err != nil {
t.Errorf("Close() = %v", err)
}
if !called {
t.Error("called after Close(); got false, wanted true")
}
if got := w.String(); got != want {
t.Errorf("w.String(); got %q, want %q", got, want)
}
}

25
internal/api/api.go Normal file
View File

@ -0,0 +1,25 @@
package api
import (
"context"
"github.com/loveuer/nf"
"nf-repo/internal/handler"
"nf-repo/internal/interfaces"
"nf-repo/internal/opt"
)
func NewApi(
ctx context.Context,
bh interfaces.BlobHandler,
uh interfaces.UploadHandler,
mh interfaces.ManifestHandler,
) *nf.App {
app := nf.New(nf.Config{BodyLimit: opt.DefaultMaxSize})
{
api := app.Group("/v2")
api.Any("/*path", handler.Root(bh, uh, mh))
}
return app
}

307
internal/handler/blobs.go Normal file
View File

@ -0,0 +1,307 @@
package handler
import (
"errors"
"fmt"
"github.com/loveuer/nf"
"github.com/sirupsen/logrus"
"io"
"log"
"net/http"
"nf-repo/internal/model"
"nf-repo/internal/opt"
"nf-repo/internal/util/rerr"
"nf-repo/internal/verify"
"path"
"strings"
)
func handleBlobs(c *nf.Ctx) error {
elem := strings.Split(c.Path(), "/")
elem = elem[1:]
if elem[len(elem)-1] == "" {
elem = elem[:len(elem)-1]
}
// Must have a path of form /v2/{name}/blobs/{upload,sha256:}
if len(elem) < 4 {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "blobs must be attached to a repo",
})
}
target := elem[len(elem)-1]
service := elem[len(elem)-2]
digest := c.Query("digest")
contentRange := c.Get("Content-Range")
rangeHeader := c.Get("Range")
repo := c.Request.URL.Host + path.Join(elem[1:len(elem)-2]...)
logrus.
WithField("handler", "handleBlob").
WithField("path", c.Path()).
WithField("method", c.Method()).
WithField("target", target).
WithField("service", service).
WithField("repo", repo).
WithField("digest", digest).
Debug()
switch c.Method() {
case http.MethodHead:
h, err := model.NewHash(target)
if err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "invalid digest",
})
}
var size int64
size, err = b.blobHandler.Stat(c.Request.Context(), repo, h)
if errors.Is(err, opt.ErrNotFound) {
return rerr.Error(c, rerr.ErrBlobUnknown)
} else if err != nil {
var re model.RedirectError
if errors.As(err, &re) {
http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
return nil
}
return rerr.Error(c, rerr.ErrInternal(err))
}
c.Set("Content-Length", fmt.Sprint(size))
c.Set("Docker-Content-Digest", h.String())
return c.SendStatus(http.StatusOK)
case http.MethodGet:
h, err := model.NewHash(target)
if err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "invalid digest",
})
}
var size int64
var r io.Reader
size, err = b.blobHandler.Stat(c.Request.Context(), repo, h)
if errors.Is(err, opt.ErrNotFound) {
return rerr.Error(c, rerr.ErrBlobUnknown)
} else if err != nil {
var re model.RedirectError
if errors.As(err, &re) {
http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
return nil
}
return rerr.Error(c, rerr.ErrInternal(err))
}
rc, err := b.blobHandler.Get(c.Request.Context(), repo, h)
if errors.Is(err, opt.ErrNotFound) {
return rerr.Error(c, rerr.ErrBlobUnknown)
} else if err != nil {
var re model.RedirectError
if errors.As(err, &re) {
http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
return nil
}
return rerr.Error(c, rerr.ErrInternal(err))
}
defer rc.Close()
r = rc
if rangeHeader != "" {
start, end := int64(0), int64(0)
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UNKNOWN",
Message: "We don't understand your Range",
})
}
n := (end + 1) - start
if ra, ok := r.(io.ReaderAt); ok {
if end+1 > size {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UNKNOWN",
Message: fmt.Sprintf("range end %d > %d size", end+1, size),
})
}
r = io.NewSectionReader(ra, start, n)
} else {
if _, err := io.CopyN(io.Discard, r, start); err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UNKNOWN",
Message: fmt.Sprintf("Failed to discard %d bytes", start),
})
}
r = io.LimitReader(r, n)
}
c.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size))
c.Set("Content-Length", fmt.Sprint(n))
c.Set("Docker-Content-Digest", h.String())
c.Status(http.StatusPartialContent)
} else {
c.Set("Content-Length", fmt.Sprint(size))
c.Set("Docker-Content-Digest", h.String())
c.Status(http.StatusOK)
}
_, err = io.Copy(c.RawWriter(), r)
return err
case http.MethodPost:
// It is weird that this is "target" instead of "service", but
// that's how the index math works out above.
if target != "uploads" {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target),
})
}
if digest != "" {
h, err := model.NewHash(digest)
if err != nil {
return rerr.Error(c, rerr.ErrDigestInvalid)
}
vrc, err := verify.ReadCloser(c.Request.Body, c.Request.ContentLength, h)
if err != nil {
return rerr.Error(c, rerr.ErrInternal(err))
}
defer vrc.Close()
if err = b.blobHandler.Put(c.Request.Context(), repo, h, vrc); err != nil {
if errors.As(err, &verify.Error{}) {
log.Printf("Digest mismatch: %v", err)
return rerr.Error(c, rerr.ErrDigestMismatch)
}
return rerr.Error(c, rerr.ErrInternal(err))
}
c.Set("Docker-Content-Digest", h.String())
return c.SendStatus(http.StatusCreated)
}
id := b.uploadHandler.UploadId()
c.Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id))
c.Set("Range", "0-0")
return c.SendStatus(http.StatusAccepted)
case http.MethodPatch:
if service != "uploads" {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service),
})
}
start, end := 0, 0
if contentRange != "" {
if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UPLOAD_UNKNOWN",
Message: "We don't understand your Content-Range",
})
}
if end != start+int(c.Request.ContentLength) {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UPLOAD_INVALID",
Message: "blob upload content range mismatch content length",
})
}
}
end = start + int(c.Request.ContentLength)
var (
length int
re *rerr.RepositoryError
)
if length, re = b.uploadHandler.Write(c.Request.Context(), target, c.Request.Body, start, end); re != nil {
return rerr.Error(c, re)
}
c.Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target))
c.Set("Range", fmt.Sprintf("0-%d", length-1))
return c.SendStatus(http.StatusNoContent)
case http.MethodPut:
if service != "uploads" {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service),
})
}
if digest == "" {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "DIGEST_INVALID",
Message: "digest not specified",
})
}
hash, err := model.NewHash(digest)
if err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "invalid digest",
})
}
var re *rerr.RepositoryError
if re = b.uploadHandler.Done(c.Request.Context(), b.blobHandler, target, c.Request.Body, int(c.Request.ContentLength), repo, hash); re != nil {
return rerr.Error(c, re)
}
c.Set("Docker-Content-Digest", hash.String())
return c.SendStatus(http.StatusCreated)
case http.MethodDelete:
h, err := model.NewHash(target)
if err != nil {
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "invalid digest",
})
}
if err := b.blobHandler.Delete(c.Request.Context(), repo, h); err != nil {
return rerr.Error(c, rerr.ErrInternal(err))
}
return c.SendStatus(http.StatusAccepted)
default:
return rerr.Error(c, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: "We don't understand your method + url",
})
}
}

37
internal/handler/catalog.go Normal file
View File

@ -0,0 +1,37 @@
package handler
import (
"github.com/loveuer/nf"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"strconv"
)
func handleCatalog(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
if ctx.Method() != "GET" {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: "We don't understand your method + url",
})
}
nStr := ctx.Query("n")
n := 10000
if nStr != "" {
n, _ = strconv.Atoi(nStr)
}
var (
re *rerr.RepositoryError
list *model.Catalog
)
if list, re = m.Catelog(ctx.Request.Context(), n, 0); re != nil {
return rerr.Error(ctx, re)
}
return ctx.JSON(list)
}

57
internal/handler/judge.go Normal file
View File

@ -0,0 +1,57 @@
package handler
import (
"github.com/loveuer/nf"
"strings"
)
func isBlob(c *nf.Ctx) bool {
elem := strings.Split(c.Path(), "/")
elem = elem[1:]
if elem[len(elem)-1] == "" {
elem = elem[:len(elem)-1]
}
if len(elem) < 3 {
return false
}
return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" &&
elem[len(elem)-2] == "uploads")
}
func isManifest(c *nf.Ctx) bool {
elems := strings.Split(c.Path(), "/")
elems = elems[1:]
if len(elems) < 4 {
return false
}
return elems[len(elems)-2] == "manifests"
}
func isTags(c *nf.Ctx) bool {
elems := strings.Split(c.Path(), "/")
elems = elems[1:]
if len(elems) < 4 {
return false
}
return elems[len(elems)-2] == "tags"
}
func isCatalog(c *nf.Ctx) bool {
elems := strings.Split(c.Path(), "/")
elems = elems[1:]
if len(elems) < 2 {
return false
}
return elems[len(elems)-1] == "_catalog"
}
func isReferrers(c *nf.Ctx) bool {
elems := strings.Split(c.Path(), "/")
elems = elems[1:]
if len(elems) < 4 {
return false
}
return elems[len(elems)-2] == "referrers"
}

127
internal/handler/manifest.go Normal file
View File

@ -0,0 +1,127 @@
package handler
import (
"bytes"
"fmt"
"github.com/loveuer/nf"
"github.com/sirupsen/logrus"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"strings"
)
func handleManifest(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
elem := strings.Split(ctx.Path(), "/")
elem = elem[1:]
target := elem[len(elem)-1]
repo := strings.Join(elem[1:len(elem)-2], "/")
logrus.
WithField("handler", "handleManifest").
WithField("path", ctx.Path()).
WithField("method", ctx.Method()).
WithField("repo", repo).
WithField("target", target).
Debug()
switch ctx.Method() {
case http.MethodGet:
var (
err error
reader io.ReadCloser
contentType string
re *rerr.RepositoryError
bs []byte
)
if reader, contentType, re = m.Get(ctx.Request.Context(), repo, target); re != nil {
return rerr.Error(ctx, re)
}
if bs, err = io.ReadAll(reader); err != nil {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusInternalServerError,
Code: "INTERNAL_SERVER_ERROR",
Message: err.Error(),
})
}
h, _, _ := model.SHA256(bytes.NewReader(bs))
ctx.Set("Docker-Content-Digest", h.String())
ctx.Set("Content-Type", contentType)
ctx.Set("Content-Length", fmt.Sprint(len(bs)))
ctx.Status(http.StatusOK)
_, err = ctx.Write(bs)
return err
case http.MethodHead:
var (
err error
reader io.ReadCloser
contentType string
re *rerr.RepositoryError
bs []byte
)
if reader, contentType, re = m.Get(ctx.Request.Context(), repo, target); re != nil {
return rerr.Error(ctx, re)
}
if bs, err = io.ReadAll(reader); err != nil {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusInternalServerError,
Code: "INTERNAL_SERVER_ERROR",
Message: err.Error(),
})
}
h, _, _ := model.SHA256(bytes.NewReader(bs))
ctx.Set("Docker-Content-Digest", h.String())
ctx.Set("Content-Type", contentType)
ctx.Set("Content-Length", fmt.Sprint(len(bs)))
return ctx.SendStatus(http.StatusOK)
case http.MethodPut:
b := &bytes.Buffer{}
io.Copy(b, ctx.Request.Body)
h, _, _ := model.SHA256(bytes.NewReader(b.Bytes()))
digest := h.String()
mf := model.Manifest{
Blob: b.Bytes(),
ContentType: ctx.Get("Content-Type"),
}
logrus.
WithField("handler", "handleManifest").
WithField("path", ctx.Path()).
WithField("method", ctx.Method()).
WithField("repo", repo).
WithField("target", target).
WithField("digest", digest).
WithField("content-type", ctx.Get("Content-Type")).
WithField("content", b.String()).
Debug()
if err := m.Put(ctx.Request.Context(), repo, target, digest, &mf); err != nil {
return rerr.Error(ctx, err)
}
ctx.Set("Docker-Content-Digest", digest)
return ctx.SendStatus(http.StatusCreated)
case http.MethodDelete:
return ctx.SendStatus(http.StatusAccepted)
default:
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: "We don't understand your method + url",
})
}
}

57
internal/handler/referrers.go Normal file
View File

@ -0,0 +1,57 @@
package handler
import (
"bytes"
"encoding/json"
"fmt"
"github.com/loveuer/nf"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/model/types"
"nf-repo/internal/util/rerr"
"strings"
)
func handleReferrers(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
if ctx.Method() != "GET" {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: "We don't understand your method + url",
})
}
elem := strings.Split(ctx.Path(), "/")
elem = elem[1:]
target := elem[len(elem)-1]
repo := strings.Join(elem[1:len(elem)-2], "/")
var (
err error
re *rerr.RepositoryError
im *model.IndexManifest
)
// Validate that incoming target is a valid digest
if _, err := model.NewHash(target); err != nil {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "UNSUPPORTED",
Message: "Target must be a valid digest",
})
}
if im, re = m.Referrers(ctx.Request.Context(), repo, target); re != nil {
return rerr.Error(ctx, re)
}
msg, _ := json.Marshal(im)
ctx.Set("Content-Length", fmt.Sprint(len(msg)))
ctx.Set("Content-Type", string(types.OCIImageIndex))
ctx.Status(http.StatusOK)
_, err = io.Copy(ctx.RawWriter(), bytes.NewReader(msg))
return err
}

46
internal/handler/root.go Normal file
View File

@ -0,0 +1,46 @@
package handler
import (
"github.com/loveuer/nf"
"nf-repo/internal/interfaces"
"nf-repo/internal/opt"
)
type blob struct {
blobHandler interfaces.BlobHandler
uploadHandler interfaces.UploadHandler
}
var (
b = &blob{}
)
func Root(bh interfaces.BlobHandler, uh interfaces.UploadHandler, mh interfaces.ManifestHandler) nf.HandlerFunc {
b.blobHandler = bh
b.uploadHandler = uh
return func(c *nf.Ctx) error {
if isBlob(c) {
return handleBlobs(c)
}
if isManifest(c) {
return handleManifest(c, mh)
}
if isTags(c) {
return handleTags(c, mh)
}
if isCatalog(c) {
return handleCatalog(c, mh)
}
if opt.ReferrersEnabled && isReferrers(c) {
return handleReferrers(c, mh)
}
c.Set("Docker-Distribution-API-Version", "registry/2.0")
return c.SendStatus(200)
}
}

50
internal/handler/tags.go Normal file
View File

@ -0,0 +1,50 @@
package handler
import (
"github.com/loveuer/nf"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"strings"
)
func handleTags(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
if ctx.Method() != "GET" {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "METHOD_UNKNOWN",
Message: "We don't understand your method + url",
})
}
type Req struct {
Last int `json:"last" query:"last"`
N int `json:"n" query:"n"`
}
elem := strings.Split(ctx.Path(), "/")
elem = elem[1:]
repo := strings.Join(elem[1:len(elem)-2], "/")
var (
err error
req = new(Req)
re *rerr.RepositoryError
list *model.Tag
)
if err = ctx.QueryParser(req); err != nil {
return rerr.Error(ctx, &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "BAD_REQUEST",
Message: err.Error(),
})
}
if list, re = m.Tags(ctx.Request.Context(), repo, req.N, req.Last); err != nil {
return rerr.Error(ctx, re)
}
return ctx.JSON(list)
}

14
internal/interfaces/blob.go Normal file
View File

@ -0,0 +1,14 @@
package interfaces
import (
"context"
"io"
"nf-repo/internal/model"
)
type BlobHandler interface {
Get(ctx context.Context, repo string, hash model.Hash) (io.ReadCloser, error)
Stat(ctx context.Context, repo string, hash model.Hash) (int64, error)
Put(ctx context.Context, repo string, hash model.Hash, rc io.ReadCloser) error
Delete(ctx context.Context, repo string, hash model.Hash) error
}

113
internal/interfaces/blobs/local.go Normal file
View File

@ -0,0 +1,113 @@
package blobs
import (
"context"
"errors"
"io"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/opt"
"os"
"path"
"sync"
)
type localHandler struct {
base string
sync.Mutex
}
func (l *localHandler) path(hash model.Hash) string {
//return path.Join(l.base, hash.Hex)
dir := path.Join(l.base, hash.Hex[:2], hash.Hex[2:4])
_ = os.MkdirAll(dir, 0755)
return path.Join(dir, hash.Hex)
}
func (l *localHandler) Get(ctx context.Context, repo string, hash model.Hash) (io.ReadCloser, error) {
var (
err error
f *os.File
)
l.Lock()
defer l.Unlock()
if f, err = os.Open(l.path(hash)); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, opt.ErrNotFound
}
return nil, err
}
return f, nil
}
func (l *localHandler) Stat(ctx context.Context, repo string, hash model.Hash) (int64, error) {
var (
err error
info os.FileInfo
)
l.Lock()
defer l.Unlock()
if info, err = os.Stat(l.path(hash)); err != nil {
if errors.Is(err, os.ErrNotExist) {
return 0, opt.ErrNotFound
}
return 0, err
}
return info.Size(), nil
}
func (l *localHandler) Put(ctx context.Context, repo string, hash model.Hash, rc io.ReadCloser) error {
var (
err error
f *os.File
)
l.Lock()
defer l.Unlock()
if f, err = os.OpenFile(l.path(hash), os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644); err != nil {
return err
}
if _, err = io.Copy(f, rc); err != nil {
return err
}
return nil
}
func (l *localHandler) Delete(ctx context.Context, repo string, hash model.Hash) error {
var (
err error
info os.FileInfo
filename = l.path(hash)
)
l.Lock()
defer l.Unlock()
if info, err = os.Stat(filename); err != nil {
if errors.Is(err, os.ErrNotExist) {
return opt.ErrNotFound
}
return err
}
_ = info
return os.Remove(filename)
}
func NewLocalBlobHandler(baseDir string) interfaces.BlobHandler {
_ = os.MkdirAll(baseDir, 0755)
return &localHandler{base: baseDir}
}

77
internal/interfaces/blobs/mem.go Normal file
View File

@ -0,0 +1,77 @@
package blobs
import (
"bytes"
"context"
"io"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/opt"
"sync"
)
type bytesCloser struct {
*bytes.Reader
}
func (r *bytesCloser) Close() error {
return nil
}
type memHandler struct {
m map[string][]byte
lock sync.Mutex
}
func NewMemBlobHandler() interfaces.BlobHandler {
return &memHandler{
m: map[string][]byte{},
}
}
func (m *memHandler) Stat(_ context.Context, _ string, h model.Hash) (int64, error) {
m.lock.Lock()
defer m.lock.Unlock()
bs, found := m.m[h.String()]
if !found {
return 0, opt.ErrNotFound
}
return int64(len(bs)), nil
}
func (m *memHandler) Get(_ context.Context, _ string, h model.Hash) (io.ReadCloser, error) {
m.lock.Lock()
defer m.lock.Unlock()
bs, found := m.m[h.String()]
if !found {
return nil, opt.ErrNotFound
}
return &bytesCloser{bytes.NewReader(bs)}, nil
}
func (m *memHandler) Put(_ context.Context, _ string, h model.Hash, rc io.ReadCloser) error {
m.lock.Lock()
defer m.lock.Unlock()
defer rc.Close()
all, err := io.ReadAll(rc)
if err != nil {
return err
}
m.m[h.String()] = all
return nil
}
func (m *memHandler) Delete(_ context.Context, _ string, h model.Hash) error {
m.lock.Lock()
defer m.lock.Unlock()
if _, found := m.m[h.String()]; !found {
return opt.ErrNotFound
}
delete(m.m, h.String())
return nil
}

10
internal/interfaces/db.go Normal file
View File

@ -0,0 +1,10 @@
package interfaces
import (
"context"
"gorm.io/gorm"
)
type Database interface {
TX(ctx context.Context) *gorm.DB
}

18
internal/interfaces/manifest.go Normal file
View File

@ -0,0 +1,18 @@
package interfaces
import (
"context"
"io"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
)
type ManifestHandler interface {
Get(ctx context.Context, repo string, target string) (io.ReadCloser, string, *rerr.RepositoryError)
Put(ctx context.Context, repo string, target string, digest string, mf *model.Manifest) *rerr.RepositoryError
Delete(ctx context.Context, repo string, target string) *rerr.RepositoryError
Catelog(ctx context.Context, limit int, last int) (*model.Catalog, *rerr.RepositoryError)
Tags(ctx context.Context, repo string, limit, last int) (*model.Tag, *rerr.RepositoryError)
Referrers(ctx context.Context, repo string, target string) (*model.IndexManifest, *rerr.RepositoryError)
}

234
internal/interfaces/manifests/db.go Normal file
View File

@ -0,0 +1,234 @@
package manifests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/samber/lo"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"nf-repo/internal/util/tools"
)
type PackageManifest struct {
Id uint64 `json:"id" gorm:"primaryKey;column:id"`
CreatedAt int64 `json:"created_at" gorm:"column:created_at;autoCreateTime:milli"`
UpdatedAt int64 `json:"updated_at" gorm:"column:updated_at;autoUpdateTime:milli"`
Repo string `json:"repo" gorm:"uniqueIndex:repo_tag_idx;column:repo"`
Target string `json:"target" gorm:"uniqueIndex:repo_tag_idx;column:target"`
Digest string `json:"digest" gorm:"unique;column:digest"`
ContentType string `json:"content_type" gorm:"column:content_type"`
Content []byte `json:"content" gorm:"column:content;type:bytes"`
}
type dbManifests struct {
db interfaces.Database
}
func (m *dbManifests) Get(ctx context.Context, repo string, target string) (io.ReadCloser, string, *rerr.RepositoryError) {
var (
err error
pm = new(PackageManifest)
h model.Hash
tx = m.db.TX(tools.Timeout(5)).Model(pm)
)
if h, err = model.NewHash(target); err == nil {
tx = tx.Where("digest", h.String())
} else {
tx = tx.Where("repo", repo).
Where("target", target)
}
if err = tx.
Take(pm).
Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, "", &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: fmt.Sprintf("Unknown name: %s@%s", repo, target),
}
}
return nil, "", rerr.ErrInternal(err)
}
return io.NopCloser(bytes.NewReader(pm.Content)), pm.ContentType, nil
}
func (m *dbManifests) Put(ctx context.Context, repo string, target string, digest string, mf *model.Manifest) *rerr.RepositoryError {
var (
err error
pm = &PackageManifest{
Repo: repo,
Target: target,
Digest: digest,
ContentType: mf.ContentType,
Content: mf.Blob,
}
)
// todo on conflict
if err = m.db.TX(tools.Timeout(5)).Create(pm).Error; err == nil {
return nil
}
logrus.
WithField("path", "dbManifests.Put.Create").
WithField("err", err.Error()).
Trace()
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
Where("(repo = ? AND target = ?) OR (digest = ?)", repo, target, digest).
Updates(map[string]any{
"repo": repo,
"target": target,
"digest": digest,
"content_type": mf.ContentType,
"content": mf.Blob,
}).
Error; err != nil {
logrus.
WithField("path", "dbManifests.Put.Updates").
WithField("err", err.Error()).
Debug()
return rerr.ErrInternal(err)
}
return nil
}
func (m *dbManifests) Delete(ctx context.Context, repo string, target string) *rerr.RepositoryError {
var (
err error
)
if err = m.db.TX(tools.Timeout(5)).
Where("repo", repo).
Where("target", target).
Delete(&PackageManifest{}).
Error; err != nil {
return rerr.ErrInternal(err)
}
return nil
}
func (m *dbManifests) Catelog(ctx context.Context, limit int, last int) (*model.Catalog, *rerr.RepositoryError) {
var (
err error
list = make([]*PackageManifest, 0)
)
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
Order("updated_at").
Offset(last).
Limit(limit).
Find(&list).
Error; err != nil {
return nil, rerr.ErrInternal(err)
}
return &model.Catalog{
Repos: lo.Map(list, func(item *PackageManifest, index int) string {
return item.Repo
}),
}, nil
}
func (m *dbManifests) Tags(ctx context.Context, repo string, limit, last int) (*model.Tag, *rerr.RepositoryError) {
var (
err error
list = make([]*PackageManifest, 0)
)
if err = m.db.TX(tools.Timeout(5)).Model(&PackageManifest{}).
Where("repo", repo).
Order("updated_at").
Offset(last).
Limit(limit).
Find(&list).
Error; err != nil {
return nil, rerr.ErrInternal(err)
}
return &model.Tag{
Name: repo,
Tags: lo.Map(list, func(item *PackageManifest, index int) string {
return item.Target
}),
}, nil
}
func (m *dbManifests) Referrers(ctx context.Context, repo string, target string) (*model.IndexManifest, *rerr.RepositoryError) {
var (
err error
pm = new(PackageManifest)
manifest = &model.IndexManifest{}
tx = m.db.TX(tools.Timeout(5)).Model(pm)
)
h, err := model.NewHash(target)
if err != nil {
tx = tx.Where("repo", repo).Where("digest", h.String())
} else {
tx = tx.Where("repo", repo).Where("target", target)
}
if err = m.db.TX(tools.Timeout(5)).Model(pm).
Take(pm).
Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: fmt.Sprintf("Unknown name: %s@%s", repo, target),
}
}
logrus.
WithField("path", "dbManifests.Referrers.Take").
WithField("repo", repo).
WithField("target", target).
WithField("err", err.Error()).
Debug()
return nil, rerr.ErrInternal(err)
}
if err = json.Unmarshal(pm.Content, manifest); err != nil {
logrus.
WithField("path", "dbManifests.Referrers.Unmarshal").
WithField("repo", repo).
WithField("target", target).
WithField("err", err.Error()).
Debug()
return nil, rerr.ErrInternal(err)
}
return manifest, nil
}
func NewManifestDBHandler(tx interfaces.Database) interfaces.ManifestHandler {
var (
err error
)
if err = tx.TX(tools.Timeout(5)).AutoMigrate(&PackageManifest{}); err != nil {
logrus.
WithField("path", "NewManifestDBHandler").
WithField("method", "AutoMigrate").
Panic(err)
}
return &dbManifests{db: tx}
}

278
internal/interfaces/manifests/mem.go Normal file
View File

@ -0,0 +1,278 @@
package manifests
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/sirupsen/logrus"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/model/types"
"nf-repo/internal/util/rerr"
"sort"
"strings"
"sync"
)
type memManifest struct {
sync.RWMutex
m map[string]map[string]*model.Manifest
}
func (m *memManifest) Referrers(ctx context.Context, repo string, target string) (*model.IndexManifest, *rerr.RepositoryError) {
m.RLock()
defer m.RUnlock()
digestToManifestMap, repoExists := m.m[repo]
if !repoExists {
return nil, &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: "Unknown name",
}
}
im := &model.IndexManifest{
SchemaVersion: 2,
MediaType: types.OCIImageIndex,
Manifests: []model.Descriptor{},
}
for digest, manifest := range digestToManifestMap {
h, err := model.NewHash(digest)
if err != nil {
continue
}
var refPointer struct {
Subject *model.Descriptor `json:"subject"`
}
json.Unmarshal(manifest.Blob, &refPointer)
if refPointer.Subject == nil {
continue
}
referenceDigest := refPointer.Subject.Digest
if referenceDigest.String() != target {
continue
}
// At this point, we know the current digest references the target
var imageAsArtifact struct {
Config struct {
MediaType string `json:"mediaType"`
} `json:"config"`
}
json.Unmarshal(manifest.Blob, &imageAsArtifact)
im.Manifests = append(im.Manifests, model.Descriptor{
MediaType: types.MediaType(manifest.ContentType),
Size: int64(len(manifest.Blob)),
Digest: h,
ArtifactType: imageAsArtifact.Config.MediaType,
})
}
return im, nil
}
func (m *memManifest) Tags(ctx context.Context, repo string, limit int, last int) (*model.Tag, *rerr.RepositoryError) {
m.RLock()
defer m.RUnlock()
c, ok := m.m[repo]
if !ok {
return nil, &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: "Unknown name",
}
}
var tags []string
for tag := range c {
if !strings.Contains(tag, "sha256:") {
tags = append(tags, tag)
}
}
sort.Strings(tags)
// https://github.com/opencontainers/distribution-spec/blob/b505e9cc53ec499edbd9c1be32298388921bb705/detail.md#tags-paginated
// Offset using last query parameter.
//if last := ctx.Query("last"); last != "" {
// for i, t := range tags {
// if t > last {
// tags = tags[i:]
// break
// }
// }
//}
//
//// Limit using n query parameter.
//if ns := ctx.Query("n"); ns != "" {
// if n, err := strconv.Atoi(ns); err != nil {
// return rerr.Error(ctx, &rerr.RepositoryError{
// Status: http.StatusBadRequest,
// Code: "BAD_REQUEST",
// Message: fmt.Sprintf("parsing n: %v", err),
// })
// } else if n < len(tags) {
// tags = tags[:n]
// }
//}
tagsToList := &model.Tag{
Name: repo,
Tags: tags,
}
return tagsToList, nil
}
func (m *memManifest) Catelog(ctx context.Context, limit, last int) (*model.Catalog, *rerr.RepositoryError) {
m.RLock()
defer m.RUnlock()
var repos []string
countRepos := 0
// TODO: implement pagination
for key := range m.m {
if countRepos >= limit {
break
}
countRepos++
repos = append(repos, key)
}
repositoriesToList := &model.Catalog{
Repos: repos,
}
return repositoriesToList, nil
}
func (m *memManifest) Put(ctx context.Context, repo string, target string, digest string, mf *model.Manifest) *rerr.RepositoryError {
// If the manifest
// list's constituent manifests are already uploaded.
// This isn't strictly required by the registry API, but some
// registries require this.
if types.MediaType(mf.ContentType).IsIndex() {
if err := func() *rerr.RepositoryError {
m.RLock()
defer m.RUnlock()
im, err := model.ParseIndexManifest(bytes.NewReader(mf.Blob))
if err != nil {
return &rerr.RepositoryError{
Status: http.StatusBadRequest,
Code: "MANIFEST_INVALID",
Message: err.Error(),
}
}
for _, desc := range im.Manifests {
if !desc.MediaType.IsDistributable() {
continue
}
if desc.MediaType.IsIndex() || desc.MediaType.IsImage() {
if _, found := m.m[repo][desc.Digest.String()]; !found {
return &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "MANIFEST_UNKNOWN",
Message: fmt.Sprintf("Sub-manifest %q not found", desc.Digest),
}
}
} else {
// TODO: Probably want to do an existence check for blobs.
logrus.Warnf("TODO: Check blobs for %q", desc.Digest)
}
}
return nil
}(); err != nil {
return err
}
}
m.Lock()
defer m.Unlock()
if _, ok := m.m[repo]; !ok {
m.m[repo] = make(map[string]*model.Manifest, 2)
}
// Allow future references by target (tag) and immutable digest.
// See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-by-digest-immutable-identifier.
m.m[repo][digest] = mf
m.m[repo][target] = mf
return nil
}
func (m *memManifest) Delete(ctx context.Context, repo string, target string) *rerr.RepositoryError {
m.Lock()
defer m.Unlock()
if _, ok := m.m[repo]; !ok {
return &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: "Unknown name",
}
}
_, ok := m.m[repo][target]
if !ok {
return &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "MANIFEST_UNKNOWN",
Message: "Unknown manifest",
}
}
delete(m.m[repo], target)
if len(m.m[repo]) == 0 {
delete(m.m, repo)
}
return nil
}
func (m *memManifest) Get(ctx context.Context, repo string, target string) (io.ReadCloser, string, *rerr.RepositoryError) {
m.RLock()
defer m.RUnlock()
c, ok := m.m[repo]
if !ok {
return nil, "", &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "NAME_UNKNOWN",
Message: "Unknown name",
}
}
f, ok := c[target]
if !ok {
return nil, "", &rerr.RepositoryError{
Status: http.StatusNotFound,
Code: "MANIFEST_UNKNOWN",
Message: "Unknown manifest",
}
}
reader := io.NopCloser(bytes.NewReader(f.Blob))
return reader, f.ContentType, nil
}
func NewManifestMemHandler() interfaces.ManifestHandler {
return &memManifest{m: make(map[string]map[string]*model.Manifest)}
}

44
internal/interfaces/tx/tx.go Normal file
View File

@ -0,0 +1,44 @@
package tx
import (
"context"
"github.com/glebarez/sqlite"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"nf-repo/internal/interfaces"
)
type tx struct {
db *gorm.DB
}
func (t *tx) TX(ctx context.Context) *gorm.DB {
return t.db.Session(&gorm.Session{}).WithContext(ctx)
}
func newTX(db *gorm.DB) interfaces.Database {
return &tx{db: db}
}
func Must(database interfaces.Database, err error) interfaces.Database {
if err != nil {
logrus.
WithField("path", "tx.Must").
WithField("err", err.Error()).
Panic()
}
if database == nil {
logrus.
WithField("path", "tx.Must").
WithField("err", "database is nil").
Panic()
}
return database
}
func NewSqliteTX(filepath string) (interfaces.Database, error) {
db, err := gorm.Open(sqlite.Open(filepath), &gorm.Config{})
return newTX(db), err
}

14
internal/interfaces/uoload.go Normal file
View File

@ -0,0 +1,14 @@
package interfaces
import (
"context"
"io"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
)
type UploadHandler interface {
UploadId() string
Write(ctx context.Context, id string, reader io.ReadCloser, start, end int) (int, *rerr.RepositoryError)
Done(ctx context.Context, bh BlobHandler, id string, closer io.ReadCloser, contentLength int, repo string, hash model.Hash) *rerr.RepositoryError
}

139
internal/interfaces/uploads/local.go Normal file
View File

@ -0,0 +1,139 @@
package uploads
import (
"context"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"nf-repo/internal/verify"
"os"
"path"
"sync"
"time"
)
type localUploader struct {
lock sync.Mutex
basedir string
sizeMap map[string]int
}
func (l *localUploader) UploadId() string {
id := fmt.Sprintf("%d", time.Now().UnixNano())
l.lock.Lock()
l.sizeMap[id] = 0
l.lock.Unlock()
return id
}
func (l *localUploader) Write(ctx context.Context, id string, reader io.ReadCloser, start, end int) (int, *rerr.RepositoryError) {
var (
err error
filename = path.Join(l.basedir, id)
f *os.File
ok bool
flag = os.O_CREATE | os.O_RDWR | os.O_TRUNC
copied int64
)
if _, ok = l.sizeMap[id]; !ok {
return 0, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UPLOAD_UNKNOWN",
Message: "Your content range doesn't match what we have",
}
}
if start > 0 {
flag = os.O_APPEND | os.O_RDWR
}
if f, err = os.OpenFile(filename, flag, 0644); err != nil {
return 0, rerr.ErrInternal(err)
}
if copied, err = io.Copy(f, reader); err != nil {
return 0, rerr.ErrInternal(err)
}
reader.Close()
l.lock.Lock()
l.sizeMap[id] += int(copied)
l.lock.Unlock()
return l.sizeMap[id], nil
}
func (l *localUploader) Done(ctx context.Context, bh interfaces.BlobHandler, id string, reader io.ReadCloser, contentLength int, repo string, hash model.Hash) *rerr.RepositoryError {
size := verify.SizeUnknown
if contentLength > 0 {
size = l.sizeMap[id] + contentLength
}
var (
err error
f *os.File
filename = path.Join(l.basedir, id)
)
if f, err = os.OpenFile(filename, os.O_RDONLY, 0644); err != nil {
return rerr.ErrInternal(err)
}
in := io.NopCloser(io.MultiReader(f, reader))
vrc, err := verify.ReadCloser(in, int64(size), hash)
if err != nil {
return rerr.ErrInternal(err)
}
defer vrc.Close()
if err := bh.Put(ctx, repo, hash, vrc); err != nil {
if errors.As(err, &verify.Error{}) {
logrus.
WithField("path", "handleBlobs.Put").
WithField("repo", repo).
WithField("hash", hash.String()).
WithField("err", fmt.Sprintf("Digest mismatch: %v", err)).Debug()
return rerr.ErrDigestMismatch
}
return rerr.ErrInternal(err)
}
delete(l.sizeMap, id)
f.Close()
if err = os.Remove(filename); err != nil {
logrus.
WithField("path", "localUploader.Done.Remove").
WithField("filename", filename).
WithField("err", err.Error()).
Warn()
}
return nil
}
func NewLocalUploader(basedir string) interfaces.UploadHandler {
var (
err error
)
if err = os.MkdirAll(basedir, 0755); err != nil {
logrus.
WithField("path", "uploads.localUploader.NewLocalUploader").
WithField("basedir", basedir).
WithField("err", err.Error()).
Panic()
}
return &localUploader{lock: sync.Mutex{}, basedir: basedir, sizeMap: make(map[string]int)}
}

101
internal/interfaces/uploads/mem.go Normal file
View File

@ -0,0 +1,101 @@
package uploads
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/sirupsen/logrus"
"io"
"net/http"
"nf-repo/internal/interfaces"
"nf-repo/internal/model"
"nf-repo/internal/util/rerr"
"nf-repo/internal/verify"
"strconv"
"sync"
"time"
)
type memUploader struct {
lock sync.Mutex
uploads map[string][]byte
}
func (m *memUploader) UploadId() string {
m.lock.Lock()
m.lock.Unlock()
id := strconv.Itoa(int(time.Now().UnixNano()))
m.uploads[id] = []byte{}
return id
}
func (m *memUploader) Write(ctx context.Context, id string, reader io.ReadCloser, start, end int) (int, *rerr.RepositoryError) {
m.lock.Lock()
defer m.lock.Unlock()
if start != len(m.uploads[id]) {
return 0, &rerr.RepositoryError{
Status: http.StatusRequestedRangeNotSatisfiable,
Code: "BLOB_UPLOAD_UNKNOWN",
Message: "Your content range doesn't match what we have",
}
}
l := bytes.NewBuffer(m.uploads[id])
size, err := io.Copy(l, reader)
if err != nil {
return 0, rerr.ErrInternal(err)
}
_ = size
m.uploads[id] = l.Bytes()
return len(m.uploads[id]), nil
}
func (m *memUploader) Done(ctx context.Context, bh interfaces.BlobHandler, id string, reader io.ReadCloser, contentLength int, repo string, hash model.Hash) *rerr.RepositoryError {
size := verify.SizeUnknown
if contentLength > 0 {
size = len(m.uploads[id]) + contentLength
}
m.lock.Lock()
defer m.lock.Unlock()
in := io.NopCloser(io.MultiReader(bytes.NewBuffer(m.uploads[id]), reader))
vrc, err := verify.ReadCloser(in, int64(size), hash)
if err != nil {
return rerr.ErrInternal(err)
}
defer vrc.Close()
if err := bh.Put(ctx, repo, hash, vrc); err != nil {
if errors.As(err, &verify.Error{}) {
logrus.
WithField("path", "handleBlobs.Put").
WithField("repo", repo).
WithField("hash", hash.String()).
WithField("err", fmt.Sprintf("Digest mismatch: %v", err)).Debug()
return rerr.ErrDigestMismatch
}
return rerr.ErrInternal(err)
}
m.uploads[id] = nil
delete(m.uploads, id)
return nil
}
func NewMemUploader() interfaces.UploadHandler {
return &memUploader{
lock: sync.Mutex{},
uploads: make(map[string][]byte),
}
}

5
internal/model/catalog.go Normal file
View File

@ -0,0 +1,5 @@
package model
type Catalog struct {
Repos []string `json:"repositories"`
}

9
internal/model/err.go Normal file
View File

@ -0,0 +1,9 @@
package model
type RedirectError struct {
// Location is the location to find the contents.
Location string
// Code is the HTTP redirect status code to return to clients.
Code int
}

76
internal/model/hash.go Normal file
View File

@ -0,0 +1,76 @@
package model
import (
"crypto"
"encoding/hex"
"fmt"
"hash"
"io"
"strings"
)
func NewHash(s string) (Hash, error) {
h := Hash{}
if err := h.parse(s); err != nil {
return Hash{}, err
}
return h, nil
}
type Hash struct {
// Algorithm holds the algorithm used to compute the hash.
Algorithm string
// Hex holds the hex portion of the content hash.
Hex string
}
func (h *Hash) parse(unquoted string) error {
parts := strings.Split(unquoted, ":")
if len(parts) != 2 {
return fmt.Errorf("cannot parse hash: %q", unquoted)
}
rest := strings.TrimLeft(parts[1], "0123456789abcdef")
if len(rest) != 0 {
return fmt.Errorf("found non-hex character in hash: %c", rest[0])
}
hasher, err := Hasher(parts[0])
if err != nil {
return err
}
// Compare the hex to the expected size (2 hex characters per byte)
if len(parts[1]) != hasher.Size()*2 {
return fmt.Errorf("wrong number of hex digits for %s: %s", parts[0], parts[1])
}
h.Algorithm = parts[0]
h.Hex = parts[1]
return nil
}
func (h Hash) String() string {
return fmt.Sprintf("%s:%s", h.Algorithm, h.Hex)
}
func Hasher(name string) (hash.Hash, error) {
switch name {
case "sha256":
return crypto.SHA256.New(), nil
default:
return nil, fmt.Errorf("unsupported hash: %q", name)
}
}
func SHA256(r io.Reader) (Hash, int64, error) {
hasher := crypto.SHA256.New()
n, err := io.Copy(hasher, r)
if err != nil {
return Hash{}, 0, err
}
return Hash{
Algorithm: "sha256",
Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))),
}, n, nil
}

39
internal/model/manifest.go Normal file
View File

@ -0,0 +1,39 @@
package model
import (
"encoding/json"
"io"
"nf-repo/internal/model/types"
)
type Manifest struct {
Blob []byte `json:"blob"`
ContentType string `json:"content_type"`
}
type Descriptor struct {
MediaType types.MediaType `json:"mediaType"`
Size int64 `json:"size"`
Digest Hash `json:"digest"`
Data []byte `json:"data,omitempty"`
URLs []string `json:"urls,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Platform *Platform `json:"platform,omitempty"`
ArtifactType string `json:"artifactType,omitempty"`
}
type IndexManifest struct {
SchemaVersion int64 `json:"schemaVersion"`
MediaType types.MediaType `json:"mediaType,omitempty"`
Manifests []Descriptor `json:"manifests"`
Annotations map[string]string `json:"annotations,omitempty"`
Subject *Descriptor `json:"subject,omitempty"`
}
func ParseIndexManifest(r io.Reader) (*IndexManifest, error) {
im := IndexManifest{}
if err := json.NewDecoder(r).Decode(&im); err != nil {
return nil, err
}
return &im, nil
}

135
internal/model/platform.go Normal file
View File

@ -0,0 +1,135 @@
package model
import (
"fmt"
"sort"
"strings"
)
// Platform represents the target os/arch for an image.
type Platform struct {
Architecture string `json:"architecture"`
OS string `json:"os"`
OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"`
Variant string `json:"variant,omitempty"`
Features []string `json:"features,omitempty"`
}
func (p Platform) String() string {
if p.OS == "" {
return ""
}
var b strings.Builder
b.WriteString(p.OS)
if p.Architecture != "" {
b.WriteString("/")
b.WriteString(p.Architecture)
}
if p.Variant != "" {
b.WriteString("/")
b.WriteString(p.Variant)
}
if p.OSVersion != "" {
b.WriteString(":")
b.WriteString(p.OSVersion)
}
return b.String()
}
// ParsePlatform parses a string representing a Platform, if possible.
func ParsePlatform(s string) (*Platform, error) {
var p Platform
parts := strings.Split(strings.TrimSpace(s), ":")
if len(parts) == 2 {
p.OSVersion = parts[1]
}
parts = strings.Split(parts[0], "/")
if len(parts) > 0 {
p.OS = parts[0]
}
if len(parts) > 1 {
p.Architecture = parts[1]
}
if len(parts) > 2 {
p.Variant = parts[2]
}
if len(parts) > 3 {
return nil, fmt.Errorf("too many slashes in platform spec: %s", s)
}
return &p, nil
}
// Equals returns true if the given platform is semantically equivalent to this one.
// The order of Features and OSFeatures is not important.
func (p Platform) Equals(o Platform) bool {
return p.OS == o.OS &&
p.Architecture == o.Architecture &&
p.Variant == o.Variant &&
p.OSVersion == o.OSVersion &&
stringSliceEqualIgnoreOrder(p.OSFeatures, o.OSFeatures) &&
stringSliceEqualIgnoreOrder(p.Features, o.Features)
}
// Satisfies returns true if this Platform "satisfies" the given spec Platform.
//
// Note that this is different from Equals and that Satisfies is not reflexive.
//
// The given spec represents "requirements" such that any missing values in the
// spec are not compared.
//
// For OSFeatures and Features, Satisfies will return true if this Platform's
// fields contain a superset of the values in the spec's fields (order ignored).
func (p Platform) Satisfies(spec Platform) bool {
return satisfies(spec.OS, p.OS) &&
satisfies(spec.Architecture, p.Architecture) &&
satisfies(spec.Variant, p.Variant) &&
satisfies(spec.OSVersion, p.OSVersion) &&
satisfiesList(spec.OSFeatures, p.OSFeatures) &&
satisfiesList(spec.Features, p.Features)
}
func satisfies(want, have string) bool {
return want == "" || want == have
}
func satisfiesList(want, have []string) bool {
if len(want) == 0 {
return true
}
set := map[string]struct{}{}
for _, h := range have {
set[h] = struct{}{}
}
for _, w := range want {
if _, ok := set[w]; !ok {
return false
}
}
return true
}
// stringSliceEqual compares 2 string slices and returns if their contents are identical.
func stringSliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, elm := range a {
if elm != b[i] {
return false
}
}
return true
}
// stringSliceEqualIgnoreOrder compares 2 string slices and returns if their contents are identical, ignoring order
func stringSliceEqualIgnoreOrder(a, b []string) bool {
if a != nil && b != nil {
sort.Strings(a)
sort.Strings(b)
}
return stringSliceEqual(a, b)
}

6
internal/model/tag.go Normal file
View File

@ -0,0 +1,6 @@
package model
type Tag struct {
Name string `json:"name"`
Tags []string `json:"tags"`
}

98
internal/model/types/types.go Normal file
View File

@ -0,0 +1,98 @@
// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package types holds common OCI media types.
package types
// MediaType is an enumeration of the supported mime types that an element of an image might have.
type MediaType string
// The collection of known MediaType values.
const (
OCIContentDescriptor MediaType = "application/vnd.oci.descriptor.v1+json"
OCIImageIndex MediaType = "application/vnd.oci.image.index.v1+json"
OCIManifestSchema1 MediaType = "application/vnd.oci.image.manifest.v1+json"
OCIConfigJSON MediaType = "application/vnd.oci.image.config.v1+json"
OCILayer MediaType = "application/vnd.oci.image.layer.v1.tar+gzip"
OCILayerZStd MediaType = "application/vnd.oci.image.layer.v1.tar+zstd"
OCIRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip"
OCIUncompressedLayer MediaType = "application/vnd.oci.image.layer.v1.tar"
OCIUncompressedRestrictedLayer MediaType = "application/vnd.oci.image.layer.nondistributable.v1.tar"
DockerManifestSchema1 MediaType = "application/vnd.docker.distribution.manifest.v1+json"
DockerManifestSchema1Signed MediaType = "application/vnd.docker.distribution.manifest.v1+prettyjws"
DockerManifestSchema2 MediaType = "application/vnd.docker.distribution.manifest.v2+json"
DockerManifestList MediaType = "application/vnd.docker.distribution.manifest.list.v2+json"
DockerLayer MediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
DockerConfigJSON MediaType = "application/vnd.docker.container.image.v1+json"
DockerPluginConfig MediaType = "application/vnd.docker.plugin.v1+json"
DockerForeignLayer MediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
DockerUncompressedLayer MediaType = "application/vnd.docker.image.rootfs.diff.tar"
OCIVendorPrefix = "vnd.oci"
DockerVendorPrefix = "vnd.docker"
)
// IsDistributable returns true if a layer is distributable, see:
// https://github.com/opencontainers/image-spec/blob/master/layer.md#non-distributable-layers
func (m MediaType) IsDistributable() bool {
switch m {
case DockerForeignLayer, OCIRestrictedLayer, OCIUncompressedRestrictedLayer:
return false
}
return true
}
// IsImage returns true if the mediaType represents an image manifest, as opposed to something else, like an index.
func (m MediaType) IsImage() bool {
switch m {
case OCIManifestSchema1, DockerManifestSchema2:
return true
}
return false
}
// IsIndex returns true if the mediaType represents an index, as opposed to something else, like an image.
func (m MediaType) IsIndex() bool {
switch m {
case OCIImageIndex, DockerManifestList:
return true
}
return false
}
// IsConfig returns true if the mediaType represents a config, as opposed to something else, like an image.
func (m MediaType) IsConfig() bool {
switch m {
case OCIConfigJSON, DockerConfigJSON:
return true
}
return false
}
func (m MediaType) IsSchema1() bool {
switch m {
case DockerManifestSchema1, DockerManifestSchema1Signed:
return true
}
return false
}
func (m MediaType) IsLayer() bool {
switch m {
case DockerLayer, DockerUncompressedLayer, OCILayer, OCILayerZStd, OCIUncompressedLayer, DockerForeignLayer, OCIRestrictedLayer, OCIUncompressedRestrictedLayer:
return true
}
return false
}

12
internal/opt/var.go Normal file
View File

@ -0,0 +1,12 @@
package opt
import "errors"
const (
DefaultMaxSize = 32 * 1024 * 1024
ReferrersEnabled = true
)
var (
ErrNotFound = errors.New("not found")
)

55
internal/util/rerr/rerr.go Normal file
View File

@ -0,0 +1,55 @@
package rerr
import (
"github.com/loveuer/nf"
"net/http"
)
type RepositoryError struct {
Status int
Code string
Message string
}
func Error(c *nf.Ctx, err *RepositoryError) error {
return c.Status(err.Status).JSON(nf.Map{
"errors": []nf.Map{
{
"code": err.Code,
"message": err.Message,
},
},
})
}
func ErrInternal(err error) *RepositoryError {
return &RepositoryError{
Status: http.StatusInternalServerError,
Code: "INTERNAL_SERVER_ERROR",
Message: err.Error(),
}
}
var ErrBlobUnknown = &RepositoryError{
Status: http.StatusNotFound,
Code: "BLOB_UNKNOWN",
Message: "Unknown blob",
}
var ErrUnsupported = &RepositoryError{
Status: http.StatusMethodNotAllowed,
Code: "UNSUPPORTED",
Message: "Unsupported operation",
}
var ErrDigestMismatch = &RepositoryError{
Status: http.StatusBadRequest,
Code: "DIGEST_INVALID",
Message: "digest does not match contents",
}
var ErrDigestInvalid = &RepositoryError{
Status: http.StatusBadRequest,
Code: "NAME_INVALID",
Message: "invalid digest",
}

22
internal/util/tools/ctx.go Normal file
View File

@ -0,0 +1,22 @@
package tools
import (
"context"
"time"
)
func Timeout(seconds ...int) (ctx context.Context) {
var (
duration time.Duration
)
if len(seconds) > 0 && seconds[0] > 0 {
duration = time.Duration(seconds[0]) * time.Second
} else {
duration = time.Duration(30) * time.Second
}
ctx, _ = context.WithTimeout(context.Background(), duration)
return
}

121
internal/verify/verify.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package verify provides a ReadCloser that verifies content matches the
// expected hash values.
package verify
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"nf-repo/internal/and"
"nf-repo/internal/model"
)
// SizeUnknown is a sentinel value to indicate that the expected size is not known.
const SizeUnknown = -1
type verifyReader struct {
inner io.Reader
hasher hash.Hash
expected model.Hash
gotSize, wantSize int64
}
// Error provides information about the failed hash verification.
type Error struct {
got string
want model.Hash
gotSize int64
}
func (v Error) Error() string {
return fmt.Sprintf("error verifying %s checksum after reading %d bytes; got %q, want %q",
v.want.Algorithm, v.gotSize, v.got, v.want)
}
// Read implements io.Reader
func (vc *verifyReader) Read(b []byte) (int, error) {
n, err := vc.inner.Read(b)
vc.gotSize += int64(n)
if err == io.EOF {
if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize {
return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize)
}
got := hex.EncodeToString(vc.hasher.Sum(nil))
if want := vc.expected.Hex; got != want {
return n, Error{
got: vc.expected.Algorithm + ":" + got,
want: vc.expected,
gotSize: vc.gotSize,
}
}
}
return n, err
}
// ReadCloser wraps the given io.ReadCloser to verify that its contents match
// the provided v1.Hash before io.EOF is returned.
//
// The reader will only be read up to size bytes, to prevent resource
// exhaustion. If EOF is returned before size bytes are read, an error is
// returned.
//
// A size of SizeUnknown (-1) indicates disables size verification when the size
// is unknown ahead of time.
func ReadCloser(r io.ReadCloser, size int64, h model.Hash) (io.ReadCloser, error) {
w, err := model.Hasher(h.Algorithm)
if err != nil {
return nil, err
}
r2 := io.TeeReader(r, w) // pass all writes to the hasher.
if size != SizeUnknown {
r2 = io.LimitReader(r2, size) // if we know the size, limit to that size.
}
return &and.ReadCloser{
Reader: &verifyReader{
inner: r2,
hasher: w,
expected: h,
wantSize: size,
},
CloseFunc: r.Close,
}, nil
}
// Descriptor verifies that the embedded Data field matches the Size and Digest
// fields of the given v1.Descriptor, returning an error if the Data field is
// missing or if it contains incorrect data.
func Descriptor(d model.Descriptor) error {
if d.Data == nil {
return errors.New("error verifying descriptor; Data == nil")
}
h, sz, err := model.SHA256(bytes.NewReader(d.Data))
if err != nil {
return err
}
if h != d.Digest {
return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest)
}
if sz != d.Size {
return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size)
}
return nil
}

147
internal/verify/verify_test.go Normal file
View File

@ -0,0 +1,147 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package verify
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"testing"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
func mustHash(s string, t *testing.T) v1.Hash {
h, _, err := v1.SHA256(strings.NewReader(s))
if err != nil {
t.Fatalf("v1.SHA256(%s) = %v", s, err)
}
t.Logf("Hashed: %q -> %q", s, h)
return h
}
func TestVerificationFailure(t *testing.T) {
want := "This is the input string."
buf := bytes.NewBufferString(want)
verified, err := ReadCloser(io.NopCloser(buf), int64(len(want)), mustHash("not the same", t))
if err != nil {
t.Fatal("ReadCloser() =", err)
}
if b, err := io.ReadAll(verified); err == nil {
t.Errorf("ReadAll() = %q; want verification error", string(b))
}
}
func TestVerification(t *testing.T) {
want := "This is the input string."
buf := bytes.NewBufferString(want)
verified, err := ReadCloser(io.NopCloser(buf), int64(len(want)), mustHash(want, t))
if err != nil {
t.Fatal("ReadCloser() =", err)
}
if _, err := io.ReadAll(verified); err != nil {
t.Error("ReadAll() =", err)
}
}
func TestVerificationSizeUnknown(t *testing.T) {
want := "This is the input string."
buf := bytes.NewBufferString(want)
verified, err := ReadCloser(io.NopCloser(buf), SizeUnknown, mustHash(want, t))
if err != nil {
t.Fatal("ReadCloser() =", err)
}
if _, err := io.ReadAll(verified); err != nil {
t.Error("ReadAll() =", err)
}
}
func TestBadHash(t *testing.T) {
h := v1.Hash{
Algorithm: "fake256",
Hex: "whatever",
}
_, err := ReadCloser(io.NopCloser(strings.NewReader("hi")), 0, h)
if err == nil {
t.Errorf("ReadCloser() = %v, wanted err", err)
}
}
func TestBadSize(t *testing.T) {
want := "This is the input string."
// having too much content or expecting too much content returns an error.
for _, size := range []int64{3, 100} {
t.Run(fmt.Sprintf("expecting size %d", size), func(t *testing.T) {
buf := bytes.NewBufferString(want)
rc, err := ReadCloser(io.NopCloser(buf), size, mustHash(want, t))
if err != nil {
t.Fatal("ReadCloser() =", err)
}
if b, err := io.ReadAll(rc); err == nil {
t.Errorf("ReadAll() = %q; want verification error", string(b))
}
})
}
}
func TestDescriptor(t *testing.T) {
for _, tc := range []struct {
err error
desc v1.Descriptor
}{{
err: errors.New("error verifying descriptor; Data == nil"),
}, {
err: errors.New(`error verifying Digest; got "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", want ":"`),
desc: v1.Descriptor{
Data: []byte("abc"),
},
}, {
err: errors.New("error verifying Size; got 3, want 0"),
desc: v1.Descriptor{
Data: []byte("abc"),
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
},
},
}, {
desc: v1.Descriptor{
Data: []byte("abc"),
Size: 3,
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
},
},
}} {
got, want := Descriptor(tc.desc), tc.err
if got == nil {
if want != nil {
t.Errorf("Descriptor(): got nil, want %v", want)
}
} else if want == nil {
t.Errorf("Descriptor(): got %v, want nil", got)
} else if got, want := got.Error(), want.Error(); got != want {
t.Errorf("Descriptor(): got %q, want %q", got, want)
}
}
}

57
internal/x/x_test.go Normal file
View File

@ -0,0 +1,57 @@
package x
import "testing"
func TestMinPathSum(t *testing.T) {
grid := [][]int{
{3, 8, 6, 0, 5, 9, 9, 6, 3, 4, 0, 5, 7, 3, 9, 3},
{0, 9, 2, 5, 5, 4, 9, 1, 4, 6, 9, 5, 6, 7, 3, 2},
{8, 2, 2, 3, 3, 3, 1, 6, 9, 1, 1, 6, 6, 2, 1, 9},
{1, 3, 6, 9, 9, 5, 0, 3, 4, 9, 1, 0, 9, 6, 2, 7},
{8, 6, 2, 2, 1, 3, 0, 0, 7, 2, 7, 5, 4, 8, 4, 8},
{4, 1, 9, 5, 8, 9, 9, 2, 0, 2, 5, 1, 8, 7, 0, 9},
{6, 2, 1, 7, 8, 1, 8, 5, 5, 7, 0, 2, 5, 7, 2, 1},
{8, 1, 7, 6, 2, 8, 1, 2, 2, 6, 4, 0, 5, 4, 1, 3},
{9, 2, 1, 7, 6, 1, 4, 3, 8, 6, 5, 5, 3, 9, 7, 3},
{0, 6, 0, 2, 4, 3, 7, 6, 1, 3, 8, 6, 9, 0, 0, 8},
{4, 3, 7, 2, 4, 3, 6, 4, 0, 3, 9, 5, 3, 6, 9, 3},
{2, 1, 8, 8, 4, 5, 6, 5, 8, 7, 3, 7, 7, 5, 8, 3},
{0, 7, 6, 6, 1, 2, 0, 3, 5, 0, 8, 0, 8, 7, 4, 3},
{0, 4, 3, 4, 9, 0, 1, 9, 7, 7, 8, 6, 4, 6, 9, 5},
{6, 5, 1, 9, 9, 2, 2, 7, 4, 2, 7, 2, 2, 3, 7, 2},
{7, 1, 9, 6, 1, 2, 7, 0, 9, 6, 6, 4, 4, 5, 1, 0},
{3, 4, 9, 2, 8, 3, 1, 2, 6, 9, 7, 0, 2, 4, 2, 0},
{5, 1, 8, 8, 4, 6, 8, 5, 2, 4, 1, 6, 2, 2, 9, 7},
}
result := minPathSum(grid)
t.Log("result:", result)
}
func minPathSum(grid [][]int) int {
return sum(grid, len(grid)-1, len(grid[0])-1)
}
func sum(grid [][]int, x, y int) int {
if x == 0 && y == 0 {
return grid[x][y]
}
if x == 0 {
return grid[x][y] + sum(grid, x, y-1)
}
if y == 0 {
return grid[x][y] + sum(grid, x-1, y)
}
return grid[x][y] + _min(sum(grid, x-1, y), sum(grid, x, y-1))
}
func _min(a, b int) int {
if a < b {
return a
}
return b
}

42
main.go Normal file
View File

@ -0,0 +1,42 @@
// https://github.com/google/go-containerregistry
package main
import (
"context"
"crypto/tls"
"github.com/sirupsen/logrus"
"nf-repo/internal/api"
"nf-repo/internal/interfaces/blobs"
"nf-repo/internal/interfaces/manifests"
"nf-repo/internal/interfaces/tx"
"nf-repo/internal/interfaces/uploads"
)
var (
tlsCfg *tls.Config
bh = blobs.NewLocalBlobHandler("images/layers")
//uh = uploads.NewMemUploader()
uh = uploads.NewLocalUploader("images/uploads")
//mh = manifests.NewManifestMemHandler()
mh = manifests.NewManifestDBHandler(tx.Must(tx.NewSqliteTX("data.sqlite")))
)
func init() {
logrus.SetFormatter(&logrus.JSONFormatter{TimestampFormat: "06/01/02 15:04:05"})
logrus.SetLevel(logrus.InfoLevel)
logrus.SetReportCaller(true)
crt, err := tls.LoadX509KeyPair("etc/repo.me.crt", "etc/repo.me.key")
if err != nil {
panic(err)
}
tlsCfg = &tls.Config{Certificates: []tls.Certificate{crt}}
}
func main() {
app := api.NewApi(context.TODO(), bh, uh, mh)
logrus.Fatal(app.RunTLS(":443", tlsCfg))
}