🔀 Merge branch 'release/v1.0.0'

.gitlab-ci.yml

@@ -0,0 +1,31 @@
+image: golang:alpine
+
+stages:
+  - test
+  - review
+
+before_script:
+  - apk add --no-cache git build-base bash make
+  - mkdir -p /go/src/gitlab.com/$CI_PROJECT_NAMESPACE /go/src/_/builds
+  - cp -r $CI_PROJECT_DIR /go/src/gitlab.com/$CI_PROJECT_PATH
+  - ln -s /go/src/gitlab.com/$CI_PROJECT_NAMESPACE /go/src/_/builds/$CI_PROJECT_NAMESPACE
+  - go get github.com/golangci/golangci-lint/cmd/golangci-lint
+  - go install github.com/golangci/golangci-lint/cmd/golangci-lint
+  - make dep
+
+unit_tests:
+  stage: test
+  script:
+    - make test
+
+code_coverage:
+  stage: test
+  script:
+    - make coverage
+  coverage: '/^coverage:\s(\d+(?:\.\d+)?%)/'
+
+lint_code:
+  stage: review
+  script:
+    - make lint
+  allow_failure: true

Makefile

@@ -0,0 +1,29 @@
+PROJECT_NAMESPACE := $(CI_PROJECT_NAMESPACE)
+PROJECT_NAME := $(CI_PROJECT_NAME)
+PROJECT_PATH := "$(PROJECT_NAMESPACE)/$(PROJECT_NAME)"
+PACKAGE_NAME := "gitlab.com/$(PROJECT_PATH)"
+PACKAGE_PATH := "$(GOPATH)/src/$(PACKAGE_NAME)"
+PACKAGE_LIST := $(shell go list $(PACKAGE_NAME)/... | grep -v /vendor/)
+GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go)
+
+.PHONY: all lint test rase coverage dep
+
+all: dep test race lint
+
+lint: ## Lint the files
+	@golangci-lint run ./...
+
+test: ## Run unittests
+	@go test -short $(PACKAGE_NAME)/...
+
+race: dep ## Run data race detector
+	@go test -race -short ${PACKAGE_LIST}
+
+coverage: ## Generate global code coverage report
+	@go test -cover -v -coverpkg=$(PACKAGE_NAME)/... ${PACKAGE_LIST}
+
+dep: ## Get the dependencies
+	@go get -v -d -t $(PACKAGE_NAME)/...
+
+help: ## Display this help screen
+	@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

README.md

@@ -1,2 +1,34 @@
 # oEmbed
-oEmbed is a format for allowing an embedded representation of a URL on third party sites. The simple API allows a website to display embedded content (such as photos or videos) when a user posts a link to that resource, without having to parse the resource directly.
+oEmbed is a format for allowing an embedded representation of a URL on third party sites. The simple API allows a website to display embedded content (such as photos or videos) when a user posts a link to that resource, without having to parse the resource directly.
+
+## Start using telegraph
+Download and install it:  
+`$ go get -u gitlab.com/toby3d/oembed`
+
+Import it in your code:  
+`import "gitlab.com/toby3d/oembed"`
+
+## Example
+```go
+package main
+
+import "gitlab.com/toby3d/oembed"
+
+var targetUrl = "https://www.youtube.com/watch?v=8jPQjjsBbIc"
+
+func main() {
+  // optional: checks what url has YouTube provider
+  if !oembed.HasProvider(targetUrl) {
+    return
+  }
+
+  // extract oEmbed object of source url
+  data, err := oembed.Extract(targetUrl)
+  if err != nil {
+    // provider not found / source not found / bad response...
+    panic(err)
+  }
+
+  // use data as you want
+}
+```

SUPPORT.md

@@ -0,0 +1,6 @@
+# [Support me on Patreon!](https://www.patreon.com/bePatron?c=243288) I develop this project in my spare time, and I do it and I will do it free of charge. However, you can make a donation or become a sponsor to make sure that I have enough coffee and pizza for night coding.
+
+**These people sponsored current version of the project:**
+- Aurielb
+- @YamiOdymel
+- MoD21k

errors.go

@@ -0,0 +1,33 @@
+package oembed
+
+import (
+	"fmt"
+
+	"golang.org/x/xerrors"
+)
+
+// Error represent a complex error
+type Error struct {
+	Message string
+	URL     string
+	Details xerrors.Frame
+}
+
+// Error returns a string formatted error
+func (e Error) Error() string {
+	return fmt.Sprint(e)
+}
+
+// Format implements fmt.Formatter method
+func (e Error) Format(f fmt.State, c rune) {
+	xerrors.FormatError(e, f, c)
+}
+
+// FormatError implements xerrors.Formatter method
+func (e Error) FormatError(p xerrors.Printer) error {
+	p.Printf("ERROR: %d [url:%s]", e.Message, e.URL)
+	if p.Detail() {
+		e.Details.Format(p)
+	}
+	return nil
+}

fetch_embed.go

@@ -0,0 +1,60 @@
+package oembed
+
+import (
+	"strconv"
+
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+	template "github.com/valyala/fasttemplate"
+)
+
+// Params represent a optional parameters for Extract method.
+type Params struct {
+	MaxWidth  int
+	MaxHeight int
+}
+
+func fetchEmbed(url string, provider *Provider, params *Params) (*OEmbed, error) {
+	resourceURL := provider.Endpoints[0].URL
+	resourceURL = template.ExecuteString(resourceURL, "{", "}", map[string]interface{}{"format": "json"})
+
+	link := http.AcquireURI()
+	defer http.ReleaseURI(link)
+	link.Update(resourceURL)
+	qa := link.QueryArgs()
+	qa.Add("format", "json")
+	qa.Add("url", url)
+
+	if params != nil && params.MaxWidth != 0 {
+		qa.Add("maxwidth", strconv.Itoa(params.MaxWidth))
+	}
+	if params != nil && params.MaxHeight != 0 {
+		qa.Add("maxheight", strconv.Itoa(params.MaxHeight))
+	}
+	link.SetQueryStringBytes(qa.QueryString())
+
+	req := http.AcquireRequest()
+	defer http.ReleaseRequest(req)
+	req.SetRequestURIBytes(link.FullURI())
+
+	resp := http.AcquireResponse()
+	defer http.ReleaseResponse(resp)
+
+	if err := http.Do(req, resp); err != nil {
+		return nil, Error{
+			Message: err.Error(),
+			URL:     url,
+		}
+	}
+
+	var oEmbed OEmbed
+	if err := json.UnmarshalFast(resp.Body(), &oEmbed); err != nil {
+		return nil, Error{
+			Message: err.Error(),
+			URL:     url,
+		}
+	}
+	oEmbed.ProviderName = provider.Name
+	oEmbed.ProviderURL = provider.URL
+	return &oEmbed, nil
+}

fetch_embed_test.go

@@ -0,0 +1,57 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetchEmbed(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := fetchEmbed(
+			"https://www.youtube.com/watch?v=8jPQjjsBbIc",
+			&Provider{
+				Name: "YouTube",
+				URL:  "https://www.youtube.com/",
+				Endpoints: []Endpoint{{
+					Schemes: []string{
+						"https://*.youtube.com/watch*",
+						"https://*.youtube.com/v/*\"",
+						"https://youtu.be/*",
+					},
+					URL:       "https://www.youtube.com/oembed",
+					Discovery: true,
+				}},
+			},
+			&Params{
+				MaxWidth:  250,
+				MaxHeight: 250,
+			},
+		)
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				provider := findProvider(url)
+				if provider == nil {
+					provider = &Provider{Endpoints: []Endpoint{Endpoint{}}}
+				}
+
+				_, err := fetchEmbed(url, provider, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}

find_provider.go

@@ -0,0 +1,82 @@
+package oembed
+
+import (
+	"regexp"
+	"strings"
+
+	http "github.com/valyala/fasthttp"
+)
+
+type providerCandidate struct {
+	Domain       string
+	ProviderName string
+	ProviderURL  string
+	Schemes      []string
+	URL          string
+}
+
+func getHostname(url string) string {
+	u := http.AcquireURI()
+	defer http.ReleaseURI(u)
+	u.Update(url)
+	if u.Host() != nil {
+		return strings.TrimPrefix(string(u.Host()), "www.")
+	}
+	return ""
+}
+
+func makeCandidate(p Provider) providerCandidate {
+	endpoint := p.Endpoints[0]
+	domain := getHostname(endpoint.URL)
+	if domain != "" {
+		domain = strings.TrimPrefix(domain, "www.")
+	} else {
+		domain = ""
+	}
+
+	return providerCandidate{
+		ProviderName: p.Name,
+		ProviderURL:  p.URL,
+		Schemes:      endpoint.Schemes,
+		URL:          endpoint.URL,
+		Domain:       domain,
+	}
+
+}
+
+func findProvider(url string) *Provider {
+	var candidates []Provider
+	for _, provider := range Providers {
+		provider := provider
+
+		endpoint := provider.Endpoints[0]
+		domain := getHostname(endpoint.URL)
+		if domain != "" {
+			domain = strings.TrimPrefix(domain, "www.")
+		} else {
+			domain = ""
+		}
+
+		if len(endpoint.Schemes) == 0 {
+			if !strings.Contains(url, domain) {
+				continue
+			}
+			candidates = append(candidates, provider)
+			continue
+		}
+		for _, scheme := range endpoint.Schemes {
+			scheme := scheme
+			reg := regexp.MustCompile(strings.Replace(scheme, "*", "(.*)", -1))
+			if !reg.MatchString(url) {
+				continue
+			}
+
+			candidates = append(candidates, provider)
+			break
+		}
+	}
+	if len(candidates) == 0 {
+		return nil
+	}
+	return &candidates[0]
+}

find_provider_test.go

@@ -0,0 +1,45 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetHostname(t *testing.T) {
+	assert := assert.New(t)
+	for k, v := range map[string]string{
+		"https://mais.uol.com.br/": "mais.uol.com.br",
+		"http://www.wootled.com/":  "wootled.com",
+		"http://yfrog.com":         "yfrog.com",
+		"https://www.youtube.com":  "youtube.com",
+		"https://www.znipe.tv":     "znipe.tv",
+		"http://":                  "",
+		"":                         "",
+	} {
+		k, v := k, v
+		t.Run(k, func(t *testing.T) { assert.Equal(v, getHostname(k)) })
+	}
+}
+
+func TestMakeCandidate(t *testing.T) {
+	assert.NotNil(t, makeCandidate(Provider{
+		Name: "YouTube",
+		URL:  "https://www.youtube.com/",
+		Endpoints: []Endpoint{
+			{
+				Schemes: []string{
+					"https://*.youtube.com/watch*",
+					"https://*.youtube.com/v/*\"",
+					"https://youtu.be/*",
+				},
+				URL:       "https://www.youtube.com/oembed",
+				Discovery: true,
+			},
+		},
+	}))
+}
+
+func TestFindProvider(t *testing.T) {
+	assert.NotNil(t, findProvider("https://www.beautiful.ai/"))
+}

is_valid_url.go

@@ -0,0 +1,8 @@
+package oembed
+
+import "net/url"
+
+func isValidURL(src string) bool {
+	_, err := url.ParseRequestURI(src)
+	return err == nil
+}

is_valid_url_test.go

@@ -0,0 +1,17 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_IsValidURL(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("invalid", func(t *testing.T) {
+		assert.False(isValidURL("str"))
+	})
+	t.Run("valid", func(t *testing.T) {
+		assert.True(isValidURL("http://www.kickstarter.com"))
+	})
+}

oembed.go

@@ -0,0 +1,35 @@
+package oembed
+
+import "golang.org/x/xerrors"
+
+// Extract try fetch oEmbed object for input url with params (if represent).
+// Return OEmbed if success.
+func Extract(url string, params *Params) (*OEmbed, error) {
+	if !isValidURL(url) {
+		return nil, Error{
+			Message: "invalid input url",
+			URL:     url,
+		}
+	}
+	if provider := findProvider(url); provider != nil {
+		resp, err := fetchEmbed(url, provider, params)
+		if err != nil {
+			return nil, Error{
+				Message: err.Error(),
+				URL:     url,
+				Details: xerrors.Caller(1),
+			}
+		}
+		return resp, nil
+	}
+
+	return nil, Error{
+		Message: "no provider found with given url",
+		URL:     url,
+	}
+}
+
+// HasProvider checks what input url has oEmbed provider
+func HasProvider(url string) bool {
+	return findProvider(url) != nil
+}

oembed_test.go

@@ -0,0 +1,46 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtract(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := Extract("https://www.youtube.com/watch?v=8jPQjjsBbIc", &Params{
+			MaxWidth:  250,
+			MaxHeight: 250,
+		})
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"",
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				_, err := Extract(url, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}
+
+func TestHasProvider(t *testing.T) {
+	t.Run("true", func(t *testing.T) {
+		assert.True(t, HasProvider("https://www.youtube.com/watch?v=8jPQjjsBbIc"))
+	})
+	t.Run("false", func(t *testing.T) {
+		assert.False(t, HasProvider("https://blog.toby3d.me/"))
+	})
+}

sync.go

@@ -0,0 +1,45 @@
+package oembed
+
+import (
+	"log"
+	"time"
+
+	"github.com/gobuffalo/packr"
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+)
+
+// SourceURL is a official url of supported providers list
+const SourceURL string = "https://oembed.com/providers.json"
+
+// Providers contains all default (or new synced) providers
+var Providers []Provider //nolint:gochecknoglobals
+
+func init() { //nolint:gochecknoinits
+	if err := Sync(SourceURL); err != nil {
+		panic(err)
+	}
+}
+
+// Sync try update Providers variable via request and parsing body of sourceURL
+func Sync(sourceURL string) error {
+	status, src, err := http.GetTimeout(nil, sourceURL, 2*time.Second)
+	if err != nil || status != http.StatusOK {
+		if src, err = packr.NewBox("./assets").Find("providers.json"); err != nil {
+			return Error{
+				Message: err.Error(),
+				URL:     sourceURL,
+			}
+		}
+	}
+
+	if err = json.Unmarshal(src, &Providers); err != nil {
+		return Error{
+			Message: err.Error(),
+			URL:     sourceURL,
+		}
+	}
+
+	log.Println("providers.json has been updated")
+	return nil
+}

sync_test.go

@@ -0,0 +1,24 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSync(t *testing.T) {
+	t.Run("invalid", func(t *testing.T) {
+		t.Run("source url", func(t *testing.T) {
+			assert.NoError(t, Sync("wtf"))
+			assert.NotZero(t, len(Providers))
+		})
+		t.Run("resource body", func(t *testing.T) {
+			assert.Error(t, Sync("https://ddg.gg/"))
+			assert.NotZero(t, len(Providers))
+		})
+	})
+	t.Run("valid url", func(t *testing.T) {
+		assert.NoError(t, Sync(SourceURL))
+		assert.NotZero(t, len(Providers))
+	})
+}

types.go

@@ -0,0 +1,108 @@
+//go:generate ffjson $GOFILE
+package oembed
+
+type (
+	// Provider represent a single provider info
+	Provider struct {
+		Name      string     `json:"provider_name"`
+		URL       string     `json:"provider_url"`
+		Endpoints []Endpoint `json:"endpoints"`
+	}
+
+	// Provider represent a single endpoint of Provider
+	Endpoint struct {
+		Schemes   []string `json:"schemes,omitempty"`
+		URL       string   `json:"url"`
+		Discovery bool     `json:"discovery,omitempty"`
+		Formats   []string `json:"formats,omitempty"`
+	}
+
+	// Response can specify a resource type, such as photo or video.
+	// Each type has specific parameters associated with it.
+	OEmbed struct {
+		// The resource type.
+		Type string `json:"type"` // required
+
+		// The oEmbed version number.
+		Version string `json:"version"` // required
+
+		// A text title, describing the resource.
+		Title string `json:"title,omitempty"`
+
+		// The name of the author/owner of the resource.
+		AuthorName string `json:"author_name,omitempty"`
+
+		// A URL for the author/owner of the resource.
+		AuthorURL string `json:"author_url,omitempty"`
+
+		// The name of the resource provider.
+		ProviderName string `json:"provider_name,omitempty"`
+
+		// The url of the resource provider.
+		ProviderURL string `json:"provider_url,omitempty"`
+
+		// The suggested cache lifetime for this resource, in seconds.
+		// Consumers may choose to use this value or not.
+		CacheAge int `json:"cache_age,omitempty"`
+
+		// A URL to a thumbnail image representing the resource.
+		// The thumbnail must respect any maxwidth and maxheight parameters.
+		// If this parameter is present, thumbnail_width and thumbnail_height must also be present.
+		ThumbnailURL string `json:"thumbnail_url,omitempty"`
+
+		// The width of the optional thumbnail.
+		// If this parameter is present, thumbnail_url and thumbnail_height must also be present.
+		ThumbnailWidth int `json:"thumbnail_width,omitempty"`
+
+		// The height of the optional thumbnail.
+		// If this parameter is present, thumbnail_url and thumbnail_width must also be present.
+		ThumbnailHeight int `json:"thumbnail_height,omitempty"`
+
+		URL string `json:"url,omitempty"`
+	}
+
+	// Photo is used for representing static photos.
+	Photo struct {
+		// The source URL of the image. Consumers should be able to insert this URL into an <img> element.
+		// Only HTTP and HTTPS URLs are valid.
+		URL string `json:"url"` // required
+
+		// The width in pixels of the image specified in the url parameter.
+		Width int `json:"width"` // required
+
+		// The height in pixels of the image specified in the url parameter.
+		Height int `json:"height"` // required
+	}
+
+	// Video is used for representing playable videos.
+	Video struct {
+		// The HTML required to embed a video player. The HTML should have no padding or margins.
+		// Consumers may wish to load the HTML in an off-domain iframe to avoid XSS vulnerabilities.
+		HTML string `json:"html"` // required
+
+		// The width in pixels required to display the HTML.
+		Width int `json:"width"` // required
+
+		// The height in pixels required to display the HTML.
+		Height int `json:"height"` // required
+	}
+
+	// Link type allow a provider to return any generic embed data (such as title and author_name), without
+	// providing either the url or html parameters. The consumer may then link to the resource, using the URL
+	// specified in the original request.
+	// Link string
+
+	// Rich is used for rich HTML content that does not fall under one of the other categories.
+	Rich struct {
+		// The HTML required to display the resource. The HTML should have no padding or margins.
+		// Consumers may wish to load the HTML in an off-domain iframe to avoid XSS vulnerabilities.
+		// The markup should be valid XHTML 1.0 Basic.
+		HTML string `json:"html"` // required
+
+		// The width in pixels required to display the HTML.
+		Width int `json:"width"` // required
+
+		// The height in pixels required to display the HTML.
+		Height int `json:"height"` // required
+	}
+)