🚧 Created request and response domains with custom unmarshling

This commit is contained in:
Maxim Lebedev 2023-09-27 22:47:04 +06:00
parent 7f5907c552
commit f37ed1a6fa
Signed by: toby3d
GPG Key ID: 1F14E25B7C119FC5
4 changed files with 906 additions and 0 deletions

2
go.mod
View File

@ -6,3 +6,5 @@ require (
github.com/google/go-cmp v0.5.9
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
)
require golang.org/x/net v0.15.0

2
go.sum
View File

@ -1,4 +1,6 @@
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=

View File

@ -0,0 +1,669 @@
package http
import (
"bytes"
"encoding/json"
"fmt"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"golang.org/x/net/html"
"source.toby3d.me/toby3d/pub/internal/common"
"source.toby3d.me/toby3d/pub/internal/domain"
)
type (
Request struct {
Action string `json:"action"`
}
RequestCreate struct {
Properties Properties `json:"properties"`
Type []string `json:"type"` // h-entry
}
RequestSource struct {
URL URL
Q string
Properties []string
}
RequestUpdate struct {
Replace *Properties `json:"replace,omitempty"`
Add *Properties `json:"add,omitempty"`
Delete *Delete `json:"delete,omitempty"`
URL URL `json:"url"`
Action string `json:"action"` // update
}
RequestDelete struct {
URL URL `json:"url"`
Action string `json:"action"`
}
RequestUndelete struct {
URL URL `json:"url"`
Action string `json:"action"`
}
ResponseSource struct {
Properties Properties `json:"properties"`
Type []string `json:"type,omitempty"`
}
Properties struct {
Audio []Figure `json:"audio,omitempty"`
Featured []URL `json:"featured,omitempty"`
InReplyTo []URL `json:"in-reply-to,omitempty"`
Like []URL `json:"like,omitempty"`
Repost []URL `json:"repost,omitempty"`
Syndication []URL `json:"syndication,omitempty"`
URL []URL `json:"url,omitempty"`
Video []Figure `json:"video,omitempty"`
Published []DateTime `json:"published,omitempty"`
Updated []DateTime `json:"updated,omitempty"`
Content []Content `json:"content,omitempty"`
Photo []Figure `json:"photo,omitempty"`
Category []string `json:"category,omitempty"`
Name []string `json:"name,omitempty"`
RSVP []string `json:"rsvp,omitempty"`
Summary []string `json:"summary,omitempty"`
UID []string `json:"uid,omitempty"`
Altitude []float32 `json:"altitude,omitempty"`
Latitude []float32 `json:"latitude,omitempty"`
Longitude []float32 `json:"longitude,omitempty"`
Duration []uint64 `json:"duration,omitempty"`
Size []uint64 `json:"size,omitempty"`
// Author []Author `json:"author,omitempty"`
// Location []Location `json:"location,omitempty"`
// LikeOf []LikeOf `json:"like-of,omitempty"`
// RepostOf []RepostOf `json:"repost-of,omitempty"`
// Comment []Comment `json:"comment,omitempty"`
// BookmarkOf []BookmarkOf `json:"bookmark-of,omitempty"`
// ListenOf []ListenOf `json:"listen-of,omitempty"`
// WatchOf []WatchOf `json:"watch-of,omitempty"`
// ReadOf []ReadOf `json:"read-of,omitempty"`
// TranslationOf []TranslationOf `json:"translation-of,omitempty"`
// Checkin []Checkin `json:"checkin,omitempty"`
// PlayOf []PlayOf `json:"play-of,omitempty"`
}
Delete struct {
Keys []string `json:"-"`
Values Properties `json:"-"`
}
Figure struct {
Value *url.URL `json:"-"`
Alt string `json:"-"`
}
Content struct {
HTML *html.Node `json:"-"`
Value string `json:"-"`
}
URL struct {
*url.URL `json:"-"`
}
DateTime struct {
time.Time `json:"-"`
}
bufferHTML struct {
HTML string `json:"html,omitempty"`
}
bufferMedia struct {
Value string `json:"value,omitempty"`
Alt string `json:"alt,omitempty"`
}
)
const MaxBodySize int64 = 100 * 1024 * 1024 // 100mb
func NewRequestCreate() *RequestCreate {
return &RequestCreate{
Type: make([]string, 0),
Properties: Properties{},
}
}
func (r *RequestCreate) bind(req *http.Request) error {
mediaType, _, err := mime.ParseMediaType(req.Header.Get(common.HeaderContentType))
if err != nil {
return fmt.Errorf("cannot understand requested Content-Type: %w", err)
}
switch mediaType {
default:
return fmt.Errorf("unsupported media type, got '%s', want '%s', '%s' or '%s'", mediaType,
common.MIMEApplicationJSON, common.MIMEMultipartForm, common.MIMEApplicationForm)
case common.MIMEApplicationJSON:
err = json.NewDecoder(req.Body).Decode(r)
case common.MIMEMultipartForm, common.MIMEApplicationForm:
in := make(map[string][]string)
switch mediaType {
case common.MIMEMultipartForm:
if err = req.ParseMultipartForm(MaxBodySize); err != nil {
return fmt.Errorf("cannot parse creation multipart body: %w", err)
}
in = req.MultipartForm.Value
case common.MIMEApplicationForm:
if err = req.ParseForm(); err != nil {
return fmt.Errorf("cannot parse creation form body: %w", err)
}
in = req.Form
}
r.Type = append(r.Type, in["h"]...)
for k, v := range in {
switch {
case strings.HasSuffix(k, "[]"):
in[strings.TrimSuffix(k, "[]")] = v
fallthrough
case strings.HasPrefix(k, "mp-"), k == "h":
delete(in, k)
}
}
// NOTE(toby3d): hack to encode URL values
var src []byte
if src, err = json.Marshal(in); err != nil {
return fmt.Errorf("cannot marshal creation values for decoding: %w", err)
}
err = json.Unmarshal(src, &r.Properties)
}
if err != nil {
return fmt.Errorf("cannot decode creation request body: %w", err)
}
for i := range r.Type {
r.Type[i] = strings.TrimPrefix(r.Type[i], "h-")
}
return nil
}
func (r *RequestCreate) populate(dst *domain.Entry) {
if len(r.Properties.Content) > 0 {
dst.Content = []byte(r.Properties.Content[0].Value)
}
if len(r.Properties.Summary) > 0 {
dst.Description = r.Properties.Summary[0]
}
if len(r.Properties.Published) > 0 {
dst.PublishedAt = r.Properties.Published[0].Time
}
if len(r.Properties.Name) > 0 {
dst.Title = r.Properties.Name[0]
}
if len(r.Properties.Updated) > 0 {
dst.UpdatedAt = r.Properties.Updated[0].Time
}
if len(r.Properties.URL) > 0 {
dst.URL = r.Properties.URL[0].URL
}
dst.Tags = append(dst.Tags, r.Properties.Category...)
for i := range r.Properties.Photo {
dst.Photo = append(dst.Photo, r.Properties.Photo[i].Value)
}
for i := range r.Properties.Syndication {
dst.Syndications = append(dst.Syndications, r.Properties.Syndication[i].URL)
}
}
func (r *RequestSource) bind(req *http.Request) error {
query := req.URL.Query()
if r.Q = query.Get("q"); !strings.EqualFold(r.Q, "source") {
return fmt.Errorf("'q' query MUST be 'source', got '%s'", r.Q)
}
var err error
if r.URL.URL, err = url.Parse(query.Get("url")); err != nil {
return fmt.Errorf("cannot unmarshal 'url' query: %w", err)
}
for k, v := range query {
if k != "properties" && k != "properties[]" {
continue
}
r.Properties = append(r.Properties, v...)
}
return nil
}
func (r *RequestUpdate) bind(req *http.Request) error {
if err := json.NewDecoder(req.Body).Decode(r); err != nil {
return fmt.Errorf("cannot decode JSON body: %w", err)
}
if !strings.EqualFold(r.Action, "update") {
return fmt.Errorf("invalid action, got '%s', want '%s'", r.Action, "update")
}
return nil
}
func (r RequestUpdate) populate(dst *domain.Entry) {
if r.Add != nil {
r.Add.CopyTo(dst)
}
if r.Replace != nil {
if len(r.Replace.Photo) > 0 {
dst.Photo = make([]*url.URL, 0)
}
if len(r.Replace.Category) > 0 {
dst.Tags = make([]string, 0)
}
if len(r.Replace.Syndication) > 0 {
dst.Syndications = make([]*url.URL, 0)
}
r.Replace.CopyTo(dst)
}
if r.Delete != nil {
r.Delete.CopyTo(dst)
}
}
func (r *RequestDelete) bind(req *http.Request) error {
mediaType, _, err := mime.ParseMediaType(req.Header.Get(common.HeaderContentType))
if err != nil {
return fmt.Errorf("cannot understand requested Content-Type: %w", err)
}
switch mediaType {
default:
return fmt.Errorf("unsupported media type, got '%s', want '%s' or '%s'", mediaType,
common.MIMEApplicationJSON, common.MIMEApplicationForm)
case common.MIMEApplicationJSON:
err = json.NewDecoder(req.Body).Decode(r)
case common.MIMEApplicationForm:
if err = req.ParseForm(); err != nil {
return fmt.Errorf("cannot decode deletion request: %w", err)
}
r.Action = req.PostForm.Get("action")
if r.URL.URL, err = url.Parse(req.PostForm.Get("url")); err != nil {
return fmt.Errorf("cannot unmarshal url in deletion request: %w", err)
}
}
if err != nil {
return fmt.Errorf("cannot parse deletion request: %w", err)
}
if !strings.EqualFold(r.Action, "delete") {
return fmt.Errorf("invalid action, got '%s', want '%s'", r.Action, "delete")
}
return nil
}
func (r *RequestUndelete) bind(req *http.Request) error {
mediaType, _, err := mime.ParseMediaType(req.Header.Get(common.HeaderContentType))
if err != nil {
return fmt.Errorf("cannot understand requested Content-Type: %w", err)
}
switch mediaType {
default:
return fmt.Errorf("unsupported media type, got '%s', want '%s' or '%s'", mediaType,
common.MIMEApplicationJSON, common.MIMEApplicationForm)
case common.MIMEApplicationJSON:
if err = json.NewDecoder(req.Body).Decode(r); err != nil {
return fmt.Errorf("cannot decode JSON body: %w", err)
}
return nil
case common.MIMEApplicationForm:
if err = req.ParseForm(); err != nil {
return fmt.Errorf("cannot parse form body: %w", err)
}
r.Action = req.PostFormValue("action")
if r.URL.URL, err = url.Parse(req.PostFormValue("url")); err != nil {
return fmt.Errorf("cannot parse url query: %w", err)
}
}
if !strings.EqualFold(r.Action, "undelete") {
return fmt.Errorf("invalid action, got '%s', want '%s'", r.Action, "undelete")
}
return nil
}
func NewResponseSource(src *domain.Entry, properties ...string) *ResponseSource {
out := &ResponseSource{
Type: make([]string, 0),
Properties: Properties{
Updated: make([]DateTime, 0),
Published: make([]DateTime, 0),
Photo: make([]Figure, 0),
Syndication: make([]URL, 0),
Content: make([]Content, 0),
Category: make([]string, 0),
Name: make([]string, 0),
Summary: make([]string, 0),
},
}
if len(properties) == 0 {
out.Type = append(out.Type, "h-entry")
properties = []string{
"updated", "published", "photo", "syndication", "content", "category", "name", "summary",
}
}
if src == nil {
return out
}
for i := range properties {
switch properties[i] {
case "updated":
if src.UpdatedAt.IsZero() {
continue
}
out.Properties.Updated = append(out.Properties.Updated, DateTime{Time: src.UpdatedAt})
case "published":
if src.PublishedAt.IsZero() {
continue
}
out.Properties.Published = append(out.Properties.Published, DateTime{
Time: src.PublishedAt,
})
case "photo":
for j := range src.Photo {
out.Properties.Photo = append(out.Properties.Photo, Figure{
Value: src.Photo[j],
})
}
case "syndication":
for j := range src.Syndications {
out.Properties.Syndication = append(out.Properties.Syndication, URL{
URL: src.Syndications[j],
})
}
case "content":
if len(src.Content) == 0 {
continue
}
out.Properties.Content = append(out.Properties.Content, Content{
Value: string(src.Content),
})
case "category":
out.Properties.Category = append(out.Properties.Category, src.Tags...)
case "name":
out.Properties.Name = append(out.Properties.Name, out.Properties.Name...)
case "summary":
if src.Description == "" {
continue
}
out.Properties.Summary = append(out.Properties.Summary, src.Description)
}
}
return out
}
func (p Properties) CopyTo(dst *domain.Entry) {
if len(p.Updated) > 0 && !p.Updated[0].IsZero() {
dst.UpdatedAt = p.Updated[0].Time
}
if len(p.Published) > 0 && !p.Published[0].IsZero() {
dst.PublishedAt = p.Published[0].Time
}
if len(p.URL) > 0 {
dst.URL = p.URL[0].URL
}
if len(p.Content) > 0 {
dst.Content = []byte(p.Content[0].Value)
}
if len(p.Name) > 0 {
dst.Title = p.Name[0]
}
if len(p.Summary) > 0 {
dst.Description = p.Summary[0]
}
for i := range p.Photo {
dst.Photo = append(dst.Photo, p.Photo[i].Value)
}
dst.Tags = append(dst.Tags, p.Category...)
for i := range p.Syndication {
dst.Syndications = append(dst.Syndications, p.Syndication[i].URL)
}
}
func (f *Figure) UnmarshalJSON(v []byte) error {
var err error
buf := new(bufferMedia)
switch v[0] {
case '"':
buf.Value, err = strconv.Unquote(string(v))
case '{':
err = json.Unmarshal(v, buf)
}
if err != nil {
return err
}
if f.Value, err = url.Parse(buf.Value); err != nil {
return err
}
f.Alt = buf.Alt
return nil
}
func (f Figure) MarshalJSON() ([]byte, error) {
if f.Value == nil {
return []byte(`""`), nil
}
return []byte(strconv.Quote(f.Value.String())), nil
}
func (d Delete) CopyTo(dst *domain.Entry) {
for i := range d.Keys {
switch d.Keys[i] {
case "category":
dst.Tags = make([]string, 0)
case "content":
dst.Content = make([]byte, 0)
case "name":
dst.Title = ""
case "photo":
dst.Photo = make([]*url.URL, 0)
case "published":
dst.PublishedAt = time.Time{}
case "summary":
dst.Description = ""
case "updated":
dst.UpdatedAt = time.Time{}
case "url":
dst.URL = new(url.URL)
case "syndication":
dst.Syndications = make([]*url.URL, 0)
}
}
// TODO(toby3d): delete property values
}
func (d *Delete) UnmarshalJSON(v []byte) error {
var err error
switch v[0] {
case '[':
err = json.Unmarshal(v, &d.Keys)
case '{':
err = json.Unmarshal(v, &d.Values)
}
return err
}
func (u *URL) UnmarshalJSON(v []byte) error {
raw, err := strconv.Unquote(string(v))
if err != nil {
return fmt.Errorf("cannot unqoute URL value: %w", err)
}
out, err := url.Parse(raw)
if err != nil {
return fmt.Errorf("cannot parse URL value: %w", err)
}
u.URL = out
return nil
}
func (u URL) MarshalJSON() ([]byte, error) {
if u.URL == nil {
return []byte(`""`), nil
}
return []byte(strconv.Quote(u.URL.String())), nil
}
func (c Content) String() string {
if c.HTML == nil {
return c.Value
}
// NOTE(toby3d): trim '<html><head></head><body>' prefix and
// '</body></html>' suffix
buf := bytes.NewBuffer(nil)
if err := html.Render(buf, c.HTML); err != nil {
return ""
}
out := buf.String()
return out[25 : len(out)-14]
}
func (c *Content) UnmarshalJSON(v []byte) error {
var err error
buf := new(bufferHTML)
switch v[0] {
case '{':
err = json.Unmarshal(v, buf)
case '"':
c.Value, err = strconv.Unquote(string(v))
}
if err != nil {
return err
}
if buf.HTML == "" {
return nil
}
if c.HTML, err = html.Parse(strings.NewReader(buf.HTML)); err != nil {
return err
}
return nil
}
func (c Content) MarshalJSON() ([]byte, error) {
if c.HTML == nil {
return []byte(strconv.Quote(c.Value)), nil
}
buf := bytes.NewBuffer(nil)
if err := html.Render(buf, c.HTML); err != nil {
return nil, err
}
out := buf.String()
// NOTE(toby3d): trim '<html><head></head><body>' prefix and
// '</body></html>' suffix
return []byte(`{"html":"` + out[25:len(out)-14] + `"}`), nil
}
func (dt *DateTime) UnmarshalJSON(b []byte) error {
v, err := strconv.Unquote(string(b))
if err != nil {
return err
}
var out time.Time
for _, format := range []string{
time.RFC3339,
// NOTE(toby3d): fallback for datetime-local input HTML node
// format
"2006-01-02T15:04",
} {
if out, err = time.Parse(format, v); err != nil {
continue
}
dt.Time = out
return nil
}
return err
}
func (dt DateTime) MarshalJSON() ([]byte, error) {
if dt.IsZero() {
return nil, nil
}
return []byte(strconv.Quote(dt.Format(time.RFC3339))), nil
}

View File

@ -0,0 +1,233 @@
package http_test
import (
"encoding/json"
"net/url"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"golang.org/x/net/html"
"source.toby3d.me/toby3d/pub/internal/entry/delivery/http"
)
type testRequest struct {
Delete *http.Delete `json:"delete,omitempty"`
Content []http.Content `json:"content,omitempty"`
Photo []*http.Figure `json:"photo,omitempty"`
}
func TestRequest(t *testing.T) {
t.Parallel()
req := new(http.Request)
if err := json.NewDecoder(strings.NewReader(`{
"action": "update",
"url": "http://example.com/",
"add": {
"syndication": ["http://web.archive.org/web/20040104110725/https://aaronpk.example/2014/06/01/9/indieweb"]
}
}`)).Decode(req); err != nil {
t.Fatal(err)
}
if req.Action != "update" {
t.Errorf("got %s, want %s", req.Action, "update")
}
}
func TestContent_UnmarshalJSON(t *testing.T) {
t.Parallel()
testContent, err := html.Parse(strings.NewReader(`<b>Hello</b> <i>World</i>`))
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
in string
out http.Content
}{{
name: "plain",
in: `"Hello World"`,
out: http.Content{
HTML: nil,
Value: "Hello World",
},
}, {
name: "html",
in: `{"html":"<b>Hello</b> <i>World</i>"}`,
out: http.Content{
HTML: testContent,
Value: "",
},
}} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
out := new(testRequest)
if err := json.Unmarshal([]byte(`{"content": [`+tc.in+`]}`), out); err != nil {
t.Fatal(err)
}
if out == nil || len(out.Content) == 0 {
t.Error("empty content result, want not nil")
return
}
if diff := cmp.Diff(out.Content[0], tc.out); diff != "" {
t.Errorf("%+s", diff)
}
})
}
}
func TestContent_MarshalJSON(t *testing.T) {
t.Parallel()
testContent, err := html.Parse(strings.NewReader(`<b>Hello</b> <i>World</i>`))
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
in http.Content
out string
name string
}{{
name: "plain",
in: http.Content{
HTML: nil,
Value: `Hello World`,
},
out: `{"content":["Hello World"]}`,
}, {
name: "html",
in: http.Content{
HTML: testContent,
Value: "",
},
out: `{"content":[{"html":"\u003cb\u003eHello\u003c/b\u003e \u003ci\u003eWorld\u003c/i\u003e"}]}`,
}, {
name: "both",
in: http.Content{
HTML: testContent,
Value: `Hello World`,
},
out: `{"content":[{"html":"\u003cb\u003eHello\u003c/b\u003e \u003ci\u003eWorld\u003c/i\u003e"}]}`,
}} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
out, err := json.Marshal(testRequest{
Content: []http.Content{tc.in},
})
if err != nil {
t.Fatal(err)
}
if string(out) != tc.out {
t.Errorf("got '%s', want '%s'", out, tc.out)
}
})
}
}
func TestDelete_UnmarshalJSON(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
in string
out http.Delete
}{{
name: "values",
in: `{"category":["indieweb"]}`,
out: http.Delete{
Keys: nil,
Values: http.Properties{
Category: []string{"indieweb"},
},
},
}, {
name: "keys",
in: `["category"]`,
out: http.Delete{
Keys: []string{"category"},
Values: http.Properties{},
},
}} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
out := new(testRequest)
if err := json.Unmarshal([]byte(`{"delete":`+tc.in+`}`), out); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(*out.Delete, tc.out); diff != "" {
t.Errorf("%+s", diff)
}
})
}
}
func TestFigure_UnmarshalJSON(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
in string
out http.Figure
}{{
name: "alt",
in: `{"value":"https://photos.example.com/globe.gif","alt":"Spinning globe animation"}`,
out: http.Figure{
Alt: "Spinning globe animation",
Value: &url.URL{
Scheme: "https",
Host: "photos.example.com",
Path: "/globe.gif",
},
},
}, {
name: "plain",
in: `"https://photos.example.com/592829482876343254.jpg"`,
out: http.Figure{
Alt: "",
Value: &url.URL{
Scheme: "https",
Host: "photos.example.com",
Path: "/592829482876343254.jpg",
},
},
}} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
out := new(testRequest)
if err := json.Unmarshal([]byte(`{"photo":[`+tc.in+`]}`), out); err != nil {
t.Fatal(err)
}
if len(out.Photo) == 0 {
t.Fatal("empty photo value, want not nil")
}
if diff := cmp.Diff(out.Photo[0], &tc.out); diff != "" {
t.Errorf("%+s", diff)
}
})
}
}