🚧 Created worked version of the package

Need documenting source code, checks by linters, add CI config

fetch_embed.go

@@ -0,0 +1,53 @@
+package oembed
+
+import (
+	"strconv"
+
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+	template "github.com/valyala/fasttemplate"
+)
+
+type Params struct {
+	MaxWidth  int
+	MaxHeight int
+}
+
+func fetchEmbed(url string, provider providerCandidate, params *Params) (*Response, error) {
+	resourceUrl := provider.URL
+	resourceUrl = template.ExecuteString(resourceUrl, "{", "}", map[string]interface{}{"format": "json"})
+
+	link := http.AcquireURI()
+	defer http.ReleaseURI(link)
+	link.Update(resourceUrl)
+	qa := link.QueryArgs()
+	qa.Add("format", "json")
+	qa.Add("url", url)
+
+	if params != nil && params.MaxWidth != 0 {
+		qa.Add("maxwidth", strconv.Itoa(params.MaxWidth))
+	}
+	if params != nil && params.MaxHeight != 0 {
+		qa.Add("maxheight", strconv.Itoa(params.MaxHeight))
+	}
+	link.SetQueryStringBytes(qa.QueryString())
+
+	req := http.AcquireRequest()
+	defer http.ReleaseRequest(req)
+	req.SetRequestURIBytes(link.FullURI())
+
+	resp := http.AcquireResponse()
+	defer http.ReleaseResponse(resp)
+
+	if err := http.Do(req, resp); err != nil {
+		return nil, err
+	}
+
+	var response Response
+	if err := json.Unmarshal(resp.Body(), &response); err != nil {
+		return nil, err
+	}
+	response.ProviderName = provider.ProviderName
+	response.ProviderURL = provider.ProviderURL
+	return &response, nil
+}

fetch_embed_test.go

@@ -0,0 +1,58 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetchEmbed(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := fetchEmbed(
+			"https://www.youtube.com/watch?v=8jPQjjsBbIc",
+			makeCandidate(Provider{
+				Name: "YouTube",
+				URL:  "https://www.youtube.com/",
+				Endpoints: []Endpoint{Endpoint{
+					Schemes: []string{
+						"https://*.youtube.com/watch*",
+						"https://*.youtube.com/v/*\"",
+						"https://youtu.be/*",
+					},
+					URL:       "https://www.youtube.com/oembed",
+					Discovery: true,
+				}},
+			}),
+			&Params{
+				MaxWidth:  250,
+				MaxHeight: 250,
+			},
+		)
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"",
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				c := findProvider(url)
+				if c == nil {
+					c = new(providerCandidate)
+				}
+
+				_, err := fetchEmbed(url, *c, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}

find_provider.go

@@ -0,0 +1,74 @@
+package oembed
+
+import (
+	"regexp"
+	"strings"
+
+	http "github.com/valyala/fasthttp"
+)
+
+type providerCandidate struct {
+	Domain       string
+	ProviderName string
+	ProviderURL  string
+	Schemes      []string
+	URL          string
+}
+
+func getHostname(url string) string {
+	u := http.AcquireURI()
+	defer http.ReleaseURI(u)
+	u.Update(url)
+	if u.Host() != nil {
+		return strings.TrimPrefix(string(u.Host()), "www.")
+	}
+	return ""
+}
+
+func makeCandidate(p Provider) providerCandidate {
+	endpoint := p.Endpoints[0]
+	domain := getHostname(endpoint.URL)
+	if domain != "" {
+		domain = strings.TrimPrefix(domain, "www.")
+	} else {
+		domain = ""
+	}
+
+	return providerCandidate{
+		ProviderName: p.Name,
+		ProviderURL:  p.URL,
+		Schemes:      endpoint.Schemes,
+		URL:          endpoint.URL,
+		Domain:       domain,
+	}
+
+}
+
+func findProvider(url string) *providerCandidate {
+	var candidates []providerCandidate
+	for _, provider := range providersList {
+		provider := provider
+		candidate := makeCandidate(provider)
+		if len(candidate.Schemes) == 0 {
+			if !strings.Contains(url, candidate.Domain) {
+				continue
+			}
+			candidates = append(candidates, candidate)
+			continue
+		}
+		for _, scheme := range candidate.Schemes {
+			scheme := scheme
+			reg := regexp.MustCompile(strings.Replace(scheme, "*", "(.*)", -1))
+			if !reg.MatchString(url) {
+				continue
+			}
+
+			candidates = append(candidates, candidate)
+			break
+		}
+	}
+	if len(candidates) == 0 {
+		return nil
+	}
+	return &candidates[0]
+}

find_provider_test.go

@@ -0,0 +1,44 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetHostname(t *testing.T) {
+	assert := assert.New(t)
+	for k, v := range map[string]string{
+		"https://mais.uol.com.br/": "mais.uol.com.br",
+		"http://www.wootled.com/":  "wootled.com",
+		"http://yfrog.com":         "yfrog.com",
+		"https://www.youtube.com":  "youtube.com",
+		"https://www.znipe.tv":     "znipe.tv",
+		"http://":                  "",
+		"":                         "",
+	} {
+		t.Run(k, func(t *testing.T) { assert.Equal(v, getHostname(k)) })
+	}
+}
+
+func TestMakeCandidate(t *testing.T) {
+	assert.NotNil(t, makeCandidate(Provider{
+		Name: "YouTube",
+		URL:  "https://www.youtube.com/",
+		Endpoints: []Endpoint{
+			Endpoint{
+				Schemes: []string{
+					"https://*.youtube.com/watch*",
+					"https://*.youtube.com/v/*\"",
+					"https://youtu.be/*",
+				},
+				URL:       "https://www.youtube.com/oembed",
+				Discovery: true,
+			},
+		},
+	}))
+}
+
+func TestFindProvider(t *testing.T) {
+	assert.NotNil(t, findProvider("https://www.beautiful.ai/"))
+}

is_valid_url.go

@@ -0,0 +1,8 @@
+package oembed
+
+import "net/url"
+
+func isValidURL(src string) bool {
+	_, err := url.ParseRequestURI(src)
+	return err == nil
+}

is_valid_url_test.go

@@ -0,0 +1,17 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_IsValidURL(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("invalid", func(t *testing.T) {
+		assert.False(isValidURL("str"))
+	})
+	t.Run("valid", func(t *testing.T) {
+		assert.True(isValidURL("http://www.kickstarter.com"))
+	})
+}

oembed.go

@@ -0,0 +1,24 @@
+package oembed
+
+import (
+	"golang.org/x/xerrors"
+)
+
+var (
+	ErrInvalidInputURL = xerrors.New("invalid input url")
+	ErrNoProviderFound = xerrors.New("no provider found with given url")
+)
+
+func Extract(url string, params *Params) (*Response, error) {
+	if !isValidURL(url) {
+		return nil, ErrInvalidInputURL
+	}
+	if c := findProvider(url); c != nil {
+		return fetchEmbed(url, *c, params)
+	}
+	return nil, ErrNoProviderFound
+}
+
+func HasProvider(url string) bool {
+	return findProvider(url) != nil
+}

oembed_test.go

@@ -0,0 +1,46 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtract(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := Extract("https://www.youtube.com/watch?v=8jPQjjsBbIc", &Params{
+			MaxWidth:  250,
+			MaxHeight: 250,
+		})
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"",
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				_, err := Extract(url, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}
+
+func TestHasProvider(t *testing.T) {
+	t.Run("true", func(t *testing.T) {
+		assert.True(t, HasProvider("https://www.youtube.com/watch?v=8jPQjjsBbIc"))
+	})
+	t.Run("false", func(t *testing.T) {
+		assert.False(t, HasProvider("https://blog.toby3d.me/"))
+	})
+}

sync.go

@@ -0,0 +1,49 @@
+package oembed
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+)
+
+const source string = "https://oembed.com/providers.json"
+
+var (
+	providersList []Provider
+
+	target = filepath.Join(
+		os.Getenv("GOPATH"), "src", "gitlab.com", "toby3d", "oembed", "assets", "providers.json",
+	)
+)
+
+func init() {
+	if err := fetch(source, target); err != nil {
+		panic(err)
+	}
+}
+
+func fetch(url, target string) error {
+	status, src, err := http.Get(nil, url)
+	if err != nil || status != http.StatusOK {
+		if src, err = ioutil.ReadFile(target); err != nil {
+			return err
+		}
+	}
+
+	if err := json.Unmarshal(src, &providersList); err != nil {
+		return err
+	}
+
+	if status == http.StatusOK {
+		if err = ioutil.WriteFile(target, src, os.ModePerm); err != nil {
+			return err
+		}
+	}
+
+	log.Println("providers.json has been updated")
+	return nil
+}

types.go

@@ -0,0 +1,106 @@
+//go:generate ffjson $GOFILE
+package oembed
+
+type (
+	Provider struct {
+		Name      string     `json:"provider_name"`
+		URL       string     `json:"provider_url"`
+		Endpoints []Endpoint `json:"endpoints"`
+	}
+
+	Endpoint struct {
+		Schemes   []string `json:"schemes,omitempty"`
+		URL       string   `json:"url"`
+		Discovery bool     `json:"discovery,omitempty"`
+		Formats   []string `json:"formats,omitempty"`
+	}
+
+	// Response can specify a resource type, such as photo or video.
+	// Each type has specific parameters associated with it.
+	Response struct {
+		// The resource type.
+		Type string `json:"type"` // required
+
+		// The oEmbed version number.
+		Version string `json:"version"` // required
+
+		// A text title, describing the resource.
+		Title string `json:"title,omitempty"`
+
+		// The name of the author/owner of the resource.
+		AuthorName string `json:"author_name,omitempty"`
+
+		// A URL for the author/owner of the resource.
+		AuthorURL string `json:"author_url,omitempty"`
+
+		// The name of the resource provider.
+		ProviderName string `json:"provider_name,omitempty"`
+
+		// The url of the resource provider.
+		ProviderURL string `json:"provider_url,omitempty"`
+
+		// The suggested cache lifetime for this resource, in seconds.
+		// Consumers may choose to use this value or not.
+		CacheAge int `json:"cache_age,omitempty"`
+
+		// A URL to a thumbnail image representing the resource.
+		// The thumbnail must respect any maxwidth and maxheight parameters.
+		// If this parameter is present, thumbnail_width and thumbnail_height must also be present.
+		ThumbnailURL string `json:"thumbnail_url,omitempty"`
+
+		// The width of the optional thumbnail.
+		// If this parameter is present, thumbnail_url and thumbnail_height must also be present.
+		ThumbnailWidth int `json:"thumbnail_width,omitempty"`
+
+		// The height of the optional thumbnail.
+		// If this parameter is present, thumbnail_url and thumbnail_width must also be present.
+		ThumbnailHeight int `json:"thumbnail_height,omitempty"`
+
+		URL string `json:"url,omitempty"`
+	}
+
+	// Photo is used for representing static photos.
+	Photo struct {
+		// The source URL of the image. Consumers should be able to insert this URL into an <img> element.
+		// Only HTTP and HTTPS URLs are valid.
+		URL string `json:"url"` // required
+
+		// The width in pixels of the image specified in the url parameter.
+		Width int `json:"width"` // required
+
+		// The height in pixels of the image specified in the url parameter.
+		Height int `json:"height"` // required
+	}
+
+	// Video is used for representing playable videos.
+	Video struct {
+		// The HTML required to embed a video player. The HTML should have no padding or margins.
+		// Consumers may wish to load the HTML in an off-domain iframe to avoid XSS vulnerabilities.
+		HTML string `json:"html"` // required
+
+		// The width in pixels required to display the HTML.
+		Width int `json:"width"` // required
+
+		// The height in pixels required to display the HTML.
+		Height int `json:"height"` // required
+	}
+
+	// Link type allow a provider to return any generic embed data (such as title and author_name), without
+	// providing either the url or html parameters. The consumer may then link to the resource, using the URL
+	// specified in the original request.
+	Link string
+
+	// Rich is used for rich HTML content that does not fall under one of the other categories.
+	Rich struct {
+		// The HTML required to display the resource. The HTML should have no padding or margins.
+		// Consumers may wish to load the HTML in an off-domain iframe to avoid XSS vulnerabilities.
+		// The markup should be valid XHTML 1.0 Basic.
+		HTML string `json:"html"` // required
+
+		// The width in pixels required to display the HTML.
+		Width int `json:"width"` // required
+
+		// The height in pixels required to display the HTML.
+		Height int `json:"height"` // required
+	}
+)