feat: mem, local uploader

internal/handler/blobs.go

@@ -0,0 +1,307 @@
+package handler
+
+import (
+	"errors"
+	"fmt"
+	"github.com/loveuer/nf"
+	"github.com/sirupsen/logrus"
+	"io"
+	"log"
+	"net/http"
+	"nf-repo/internal/model"
+	"nf-repo/internal/opt"
+	"nf-repo/internal/util/rerr"
+	"nf-repo/internal/verify"
+	"path"
+	"strings"
+)
+
+func handleBlobs(c *nf.Ctx) error {
+	elem := strings.Split(c.Path(), "/")
+	elem = elem[1:]
+	if elem[len(elem)-1] == "" {
+		elem = elem[:len(elem)-1]
+	}
+
+	// Must have a path of form /v2/{name}/blobs/{upload,sha256:}
+	if len(elem) < 4 {
+		return rerr.Error(c, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "NAME_INVALID",
+			Message: "blobs must be attached to a repo",
+		})
+	}
+
+	target := elem[len(elem)-1]
+	service := elem[len(elem)-2]
+	digest := c.Query("digest")
+	contentRange := c.Get("Content-Range")
+	rangeHeader := c.Get("Range")
+	repo := c.Request.URL.Host + path.Join(elem[1:len(elem)-2]...)
+
+	logrus.
+		WithField("handler", "handleBlob").
+		WithField("path", c.Path()).
+		WithField("method", c.Method()).
+		WithField("target", target).
+		WithField("service", service).
+		WithField("repo", repo).
+		WithField("digest", digest).
+		Debug()
+
+	switch c.Method() {
+	case http.MethodHead:
+		h, err := model.NewHash(target)
+		if err != nil {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "NAME_INVALID",
+				Message: "invalid digest",
+			})
+		}
+
+		var size int64
+
+		size, err = b.blobHandler.Stat(c.Request.Context(), repo, h)
+		if errors.Is(err, opt.ErrNotFound) {
+			return rerr.Error(c, rerr.ErrBlobUnknown)
+		} else if err != nil {
+			var re model.RedirectError
+			if errors.As(err, &re) {
+				http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
+				return nil
+			}
+			return rerr.Error(c, rerr.ErrInternal(err))
+		}
+
+		c.Set("Content-Length", fmt.Sprint(size))
+		c.Set("Docker-Content-Digest", h.String())
+		return c.SendStatus(http.StatusOK)
+
+	case http.MethodGet:
+		h, err := model.NewHash(target)
+		if err != nil {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "NAME_INVALID",
+				Message: "invalid digest",
+			})
+		}
+
+		var size int64
+		var r io.Reader
+
+		size, err = b.blobHandler.Stat(c.Request.Context(), repo, h)
+		if errors.Is(err, opt.ErrNotFound) {
+			return rerr.Error(c, rerr.ErrBlobUnknown)
+		} else if err != nil {
+			var re model.RedirectError
+			if errors.As(err, &re) {
+				http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
+				return nil
+			}
+
+			return rerr.Error(c, rerr.ErrInternal(err))
+		}
+
+		rc, err := b.blobHandler.Get(c.Request.Context(), repo, h)
+		if errors.Is(err, opt.ErrNotFound) {
+			return rerr.Error(c, rerr.ErrBlobUnknown)
+		} else if err != nil {
+			var re model.RedirectError
+			if errors.As(err, &re) {
+				http.Redirect(c.RawWriter(), c.Request, re.Location, re.Code)
+				return nil
+			}
+
+			return rerr.Error(c, rerr.ErrInternal(err))
+		}
+
+		defer rc.Close()
+		r = rc
+
+		if rangeHeader != "" {
+			start, end := int64(0), int64(0)
+			if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil {
+				return rerr.Error(c, &rerr.RepositoryError{
+					Status:  http.StatusRequestedRangeNotSatisfiable,
+					Code:    "BLOB_UNKNOWN",
+					Message: "We don't understand your Range",
+				})
+			}
+
+			n := (end + 1) - start
+			if ra, ok := r.(io.ReaderAt); ok {
+				if end+1 > size {
+					return rerr.Error(c, &rerr.RepositoryError{
+						Status:  http.StatusRequestedRangeNotSatisfiable,
+						Code:    "BLOB_UNKNOWN",
+						Message: fmt.Sprintf("range end %d > %d size", end+1, size),
+					})
+				}
+				r = io.NewSectionReader(ra, start, n)
+			} else {
+				if _, err := io.CopyN(io.Discard, r, start); err != nil {
+					return rerr.Error(c, &rerr.RepositoryError{
+						Status:  http.StatusRequestedRangeNotSatisfiable,
+						Code:    "BLOB_UNKNOWN",
+						Message: fmt.Sprintf("Failed to discard %d bytes", start),
+					})
+				}
+
+				r = io.LimitReader(r, n)
+			}
+
+			c.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size))
+			c.Set("Content-Length", fmt.Sprint(n))
+			c.Set("Docker-Content-Digest", h.String())
+			c.Status(http.StatusPartialContent)
+		} else {
+			c.Set("Content-Length", fmt.Sprint(size))
+			c.Set("Docker-Content-Digest", h.String())
+			c.Status(http.StatusOK)
+		}
+
+		_, err = io.Copy(c.RawWriter(), r)
+		return err
+
+	case http.MethodPost:
+		// It is weird that this is "target" instead of "service", but
+		// that's how the index math works out above.
+		if target != "uploads" {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "METHOD_UNKNOWN",
+				Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target),
+			})
+		}
+
+		if digest != "" {
+			h, err := model.NewHash(digest)
+			if err != nil {
+				return rerr.Error(c, rerr.ErrDigestInvalid)
+			}
+
+			vrc, err := verify.ReadCloser(c.Request.Body, c.Request.ContentLength, h)
+			if err != nil {
+				return rerr.Error(c, rerr.ErrInternal(err))
+			}
+			defer vrc.Close()
+
+			if err = b.blobHandler.Put(c.Request.Context(), repo, h, vrc); err != nil {
+				if errors.As(err, &verify.Error{}) {
+					log.Printf("Digest mismatch: %v", err)
+					return rerr.Error(c, rerr.ErrDigestMismatch)
+				}
+				return rerr.Error(c, rerr.ErrInternal(err))
+			}
+			c.Set("Docker-Content-Digest", h.String())
+			return c.SendStatus(http.StatusCreated)
+		}
+
+		id := b.uploadHandler.UploadId()
+
+		c.Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id))
+		c.Set("Range", "0-0")
+
+		return c.SendStatus(http.StatusAccepted)
+
+	case http.MethodPatch:
+		if service != "uploads" {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "METHOD_UNKNOWN",
+				Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service),
+			})
+		}
+
+		start, end := 0, 0
+
+		if contentRange != "" {
+			if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil {
+				return rerr.Error(c, &rerr.RepositoryError{
+					Status:  http.StatusRequestedRangeNotSatisfiable,
+					Code:    "BLOB_UPLOAD_UNKNOWN",
+					Message: "We don't understand your Content-Range",
+				})
+			}
+
+			if end != start+int(c.Request.ContentLength) {
+				return rerr.Error(c, &rerr.RepositoryError{
+					Status:  http.StatusRequestedRangeNotSatisfiable,
+					Code:    "BLOB_UPLOAD_INVALID",
+					Message: "blob upload content range mismatch content length",
+				})
+			}
+		}
+
+		end = start + int(c.Request.ContentLength)
+
+		var (
+			length int
+			re     *rerr.RepositoryError
+		)
+		if length, re = b.uploadHandler.Write(c.Request.Context(), target, c.Request.Body, start, end); re != nil {
+			return rerr.Error(c, re)
+		}
+
+		c.Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target))
+		c.Set("Range", fmt.Sprintf("0-%d", length-1))
+		return c.SendStatus(http.StatusNoContent)
+
+	case http.MethodPut:
+		if service != "uploads" {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "METHOD_UNKNOWN",
+				Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service),
+			})
+		}
+
+		if digest == "" {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "DIGEST_INVALID",
+				Message: "digest not specified",
+			})
+		}
+
+		hash, err := model.NewHash(digest)
+		if err != nil {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "NAME_INVALID",
+				Message: "invalid digest",
+			})
+		}
+
+		var re *rerr.RepositoryError
+		if re = b.uploadHandler.Done(c.Request.Context(), b.blobHandler, target, c.Request.Body, int(c.Request.ContentLength), repo, hash); re != nil {
+			return rerr.Error(c, re)
+		}
+
+		c.Set("Docker-Content-Digest", hash.String())
+		return c.SendStatus(http.StatusCreated)
+
+	case http.MethodDelete:
+		h, err := model.NewHash(target)
+		if err != nil {
+			return rerr.Error(c, &rerr.RepositoryError{
+				Status:  http.StatusBadRequest,
+				Code:    "NAME_INVALID",
+				Message: "invalid digest",
+			})
+		}
+		if err := b.blobHandler.Delete(c.Request.Context(), repo, h); err != nil {
+			return rerr.Error(c, rerr.ErrInternal(err))
+		}
+		return c.SendStatus(http.StatusAccepted)
+
+	default:
+		return rerr.Error(c, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "METHOD_UNKNOWN",
+			Message: "We don't understand your method + url",
+		})
+	}
+}

internal/handler/catalog.go

@@ -0,0 +1,37 @@
+package handler
+
+import (
+	"github.com/loveuer/nf"
+	"net/http"
+	"nf-repo/internal/interfaces"
+	"nf-repo/internal/model"
+	"nf-repo/internal/util/rerr"
+	"strconv"
+)
+
+func handleCatalog(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
+	if ctx.Method() != "GET" {
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "METHOD_UNKNOWN",
+			Message: "We don't understand your method + url",
+		})
+	}
+
+	nStr := ctx.Query("n")
+	n := 10000
+	if nStr != "" {
+		n, _ = strconv.Atoi(nStr)
+	}
+
+	var (
+		re   *rerr.RepositoryError
+		list *model.Catalog
+	)
+
+	if list, re = m.Catelog(ctx.Request.Context(), n, 0); re != nil {
+		return rerr.Error(ctx, re)
+	}
+
+	return ctx.JSON(list)
+}

internal/handler/judge.go

@@ -0,0 +1,57 @@
+package handler
+
+import (
+	"github.com/loveuer/nf"
+	"strings"
+)
+
+func isBlob(c *nf.Ctx) bool {
+	elem := strings.Split(c.Path(), "/")
+	elem = elem[1:]
+	if elem[len(elem)-1] == "" {
+		elem = elem[:len(elem)-1]
+	}
+	if len(elem) < 3 {
+		return false
+	}
+	return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" &&
+		elem[len(elem)-2] == "uploads")
+}
+
+func isManifest(c *nf.Ctx) bool {
+	elems := strings.Split(c.Path(), "/")
+	elems = elems[1:]
+	if len(elems) < 4 {
+		return false
+	}
+	return elems[len(elems)-2] == "manifests"
+}
+
+func isTags(c *nf.Ctx) bool {
+	elems := strings.Split(c.Path(), "/")
+	elems = elems[1:]
+	if len(elems) < 4 {
+		return false
+	}
+	return elems[len(elems)-2] == "tags"
+}
+
+func isCatalog(c *nf.Ctx) bool {
+	elems := strings.Split(c.Path(), "/")
+	elems = elems[1:]
+	if len(elems) < 2 {
+		return false
+	}
+
+	return elems[len(elems)-1] == "_catalog"
+}
+
+func isReferrers(c *nf.Ctx) bool {
+	elems := strings.Split(c.Path(), "/")
+	elems = elems[1:]
+	if len(elems) < 4 {
+		return false
+	}
+
+	return elems[len(elems)-2] == "referrers"
+}

internal/handler/manifest.go

@@ -0,0 +1,127 @@
+package handler
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/loveuer/nf"
+	"github.com/sirupsen/logrus"
+	"io"
+	"net/http"
+	"nf-repo/internal/interfaces"
+	"nf-repo/internal/model"
+	"nf-repo/internal/util/rerr"
+	"strings"
+)
+
+func handleManifest(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
+	elem := strings.Split(ctx.Path(), "/")
+	elem = elem[1:]
+	target := elem[len(elem)-1]
+	repo := strings.Join(elem[1:len(elem)-2], "/")
+
+	logrus.
+		WithField("handler", "handleManifest").
+		WithField("path", ctx.Path()).
+		WithField("method", ctx.Method()).
+		WithField("repo", repo).
+		WithField("target", target).
+		Debug()
+
+	switch ctx.Method() {
+	case http.MethodGet:
+		var (
+			err         error
+			reader      io.ReadCloser
+			contentType string
+			re          *rerr.RepositoryError
+			bs          []byte
+		)
+
+		if reader, contentType, re = m.Get(ctx.Request.Context(), repo, target); re != nil {
+			return rerr.Error(ctx, re)
+		}
+
+		if bs, err = io.ReadAll(reader); err != nil {
+			return rerr.Error(ctx, &rerr.RepositoryError{
+				Status:  http.StatusInternalServerError,
+				Code:    "INTERNAL_SERVER_ERROR",
+				Message: err.Error(),
+			})
+		}
+
+		h, _, _ := model.SHA256(bytes.NewReader(bs))
+		ctx.Set("Docker-Content-Digest", h.String())
+		ctx.Set("Content-Type", contentType)
+		ctx.Set("Content-Length", fmt.Sprint(len(bs)))
+		ctx.Status(http.StatusOK)
+		_, err = ctx.Write(bs)
+		return err
+
+	case http.MethodHead:
+		var (
+			err         error
+			reader      io.ReadCloser
+			contentType string
+			re          *rerr.RepositoryError
+			bs          []byte
+		)
+
+		if reader, contentType, re = m.Get(ctx.Request.Context(), repo, target); re != nil {
+			return rerr.Error(ctx, re)
+		}
+
+		if bs, err = io.ReadAll(reader); err != nil {
+			return rerr.Error(ctx, &rerr.RepositoryError{
+				Status:  http.StatusInternalServerError,
+				Code:    "INTERNAL_SERVER_ERROR",
+				Message: err.Error(),
+			})
+		}
+
+		h, _, _ := model.SHA256(bytes.NewReader(bs))
+		ctx.Set("Docker-Content-Digest", h.String())
+		ctx.Set("Content-Type", contentType)
+		ctx.Set("Content-Length", fmt.Sprint(len(bs)))
+
+		return ctx.SendStatus(http.StatusOK)
+
+	case http.MethodPut:
+		b := &bytes.Buffer{}
+		io.Copy(b, ctx.Request.Body)
+		h, _, _ := model.SHA256(bytes.NewReader(b.Bytes()))
+		digest := h.String()
+
+		mf := model.Manifest{
+			Blob:        b.Bytes(),
+			ContentType: ctx.Get("Content-Type"),
+		}
+
+		logrus.
+			WithField("handler", "handleManifest").
+			WithField("path", ctx.Path()).
+			WithField("method", ctx.Method()).
+			WithField("repo", repo).
+			WithField("target", target).
+			WithField("digest", digest).
+			WithField("content-type", ctx.Get("Content-Type")).
+			WithField("content", b.String()).
+			Debug()
+
+		if err := m.Put(ctx.Request.Context(), repo, target, digest, &mf); err != nil {
+			return rerr.Error(ctx, err)
+		}
+
+		ctx.Set("Docker-Content-Digest", digest)
+		return ctx.SendStatus(http.StatusCreated)
+
+	case http.MethodDelete:
+		return ctx.SendStatus(http.StatusAccepted)
+
+	default:
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "METHOD_UNKNOWN",
+			Message: "We don't understand your method + url",
+		})
+	}
+}

internal/handler/referrers.go

@@ -0,0 +1,57 @@
+package handler
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/loveuer/nf"
+	"io"
+	"net/http"
+	"nf-repo/internal/interfaces"
+	"nf-repo/internal/model"
+	"nf-repo/internal/model/types"
+	"nf-repo/internal/util/rerr"
+	"strings"
+)
+
+func handleReferrers(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
+	if ctx.Method() != "GET" {
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "METHOD_UNKNOWN",
+			Message: "We don't understand your method + url",
+		})
+	}
+
+	elem := strings.Split(ctx.Path(), "/")
+	elem = elem[1:]
+	target := elem[len(elem)-1]
+	repo := strings.Join(elem[1:len(elem)-2], "/")
+
+	var (
+		err error
+		re  *rerr.RepositoryError
+		im  *model.IndexManifest
+	)
+
+	// Validate that incoming target is a valid digest
+	if _, err := model.NewHash(target); err != nil {
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "UNSUPPORTED",
+			Message: "Target must be a valid digest",
+		})
+	}
+
+	if im, re = m.Referrers(ctx.Request.Context(), repo, target); re != nil {
+		return rerr.Error(ctx, re)
+	}
+
+	msg, _ := json.Marshal(im)
+	ctx.Set("Content-Length", fmt.Sprint(len(msg)))
+	ctx.Set("Content-Type", string(types.OCIImageIndex))
+	ctx.Status(http.StatusOK)
+	_, err = io.Copy(ctx.RawWriter(), bytes.NewReader(msg))
+
+	return err
+}

internal/handler/root.go

@@ -0,0 +1,46 @@
+package handler
+
+import (
+	"github.com/loveuer/nf"
+	"nf-repo/internal/interfaces"
+	"nf-repo/internal/opt"
+)
+
+type blob struct {
+	blobHandler   interfaces.BlobHandler
+	uploadHandler interfaces.UploadHandler
+}
+
+var (
+	b = &blob{}
+)
+
+func Root(bh interfaces.BlobHandler, uh interfaces.UploadHandler, mh interfaces.ManifestHandler) nf.HandlerFunc {
+	b.blobHandler = bh
+	b.uploadHandler = uh
+	return func(c *nf.Ctx) error {
+		if isBlob(c) {
+			return handleBlobs(c)
+		}
+
+		if isManifest(c) {
+			return handleManifest(c, mh)
+		}
+
+		if isTags(c) {
+			return handleTags(c, mh)
+		}
+
+		if isCatalog(c) {
+			return handleCatalog(c, mh)
+		}
+
+		if opt.ReferrersEnabled && isReferrers(c) {
+			return handleReferrers(c, mh)
+		}
+
+		c.Set("Docker-Distribution-API-Version", "registry/2.0")
+
+		return c.SendStatus(200)
+	}
+}

internal/handler/tags.go

@@ -0,0 +1,50 @@
+package handler
+
+import (
+	"github.com/loveuer/nf"
+	"net/http"
+	"nf-repo/internal/interfaces"
+	"nf-repo/internal/model"
+	"nf-repo/internal/util/rerr"
+	"strings"
+)
+
+func handleTags(ctx *nf.Ctx, m interfaces.ManifestHandler) error {
+	if ctx.Method() != "GET" {
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "METHOD_UNKNOWN",
+			Message: "We don't understand your method + url",
+		})
+	}
+
+	type Req struct {
+		Last int `json:"last" query:"last"`
+		N    int `json:"n" query:"n"`
+	}
+
+	elem := strings.Split(ctx.Path(), "/")
+	elem = elem[1:]
+	repo := strings.Join(elem[1:len(elem)-2], "/")
+
+	var (
+		err  error
+		req  = new(Req)
+		re   *rerr.RepositoryError
+		list *model.Tag
+	)
+
+	if err = ctx.QueryParser(req); err != nil {
+		return rerr.Error(ctx, &rerr.RepositoryError{
+			Status:  http.StatusBadRequest,
+			Code:    "BAD_REQUEST",
+			Message: err.Error(),
+		})
+	}
+
+	if list, re = m.Tags(ctx.Request.Context(), repo, req.N, req.Last); err != nil {
+		return rerr.Error(ctx, re)
+	}
+
+	return ctx.JSON(list)
+}