♻️ Refactored source code, packed providers.json

This commit is contained in:
Maxim Lebedev 2019-05-24 14:29:36 +05:00
parent 22aed18257
commit 6b9a7b7bf5
No known key found for this signature in database
GPG Key ID: F8978F46FF0FFA4F
11 changed files with 919 additions and 827 deletions

11
a_oembed-packr.go Normal file

File diff suppressed because one or more lines are too long

33
errors.go Normal file
View File

@ -0,0 +1,33 @@
package oembed
import (
"fmt"
"golang.org/x/xerrors"
)
// Error represent a complex error
type Error struct {
Message string
URL string
Details xerrors.Frame
}
// Error returns a string formatted error
func (e Error) Error() string {
return fmt.Sprint(e)
}
// Format implements fmt.Formatter method
func (e Error) Format(f fmt.State, c rune) {
xerrors.FormatError(e, f, c)
}
// FormatError implements xerrors.Formatter method
func (e Error) FormatError(p xerrors.Printer) error {
p.Printf("ERROR: %d [url:%s]", e.Message, e.URL)
if p.Detail() {
e.Details.Format(p)
}
return nil
}

View File

@ -8,18 +8,19 @@ import (
template "github.com/valyala/fasttemplate"
)
// Params represent a optional parameters for Extract method.
type Params struct {
MaxWidth int
MaxHeight int
}
func fetchEmbed(url string, provider providerCandidate, params *Params) (*Response, error) {
resourceUrl := provider.URL
resourceUrl = template.ExecuteString(resourceUrl, "{", "}", map[string]interface{}{"format": "json"})
func fetchEmbed(url string, provider *Provider, params *Params) (*OEmbed, error) {
resourceURL := provider.Endpoints[0].URL
resourceURL = template.ExecuteString(resourceURL, "{", "}", map[string]interface{}{"format": "json"})
link := http.AcquireURI()
defer http.ReleaseURI(link)
link.Update(resourceUrl)
link.Update(resourceURL)
qa := link.QueryArgs()
qa.Add("format", "json")
qa.Add("url", url)
@ -40,14 +41,20 @@ func fetchEmbed(url string, provider providerCandidate, params *Params) (*Respon
defer http.ReleaseResponse(resp)
if err := http.Do(req, resp); err != nil {
return nil, err
return nil, Error{
Message: err.Error(),
URL: url,
}
}
var response Response
if err := json.Unmarshal(resp.Body(), &response); err != nil {
return nil, err
var oEmbed OEmbed
if err := json.UnmarshalFast(resp.Body(), &oEmbed); err != nil {
return nil, Error{
Message: err.Error(),
URL: url,
}
}
response.ProviderName = provider.ProviderName
response.ProviderURL = provider.ProviderURL
return &response, nil
oEmbed.ProviderName = provider.Name
oEmbed.ProviderURL = provider.URL
return &oEmbed, nil
}

View File

@ -11,10 +11,10 @@ func TestFetchEmbed(t *testing.T) {
t.Run("valid", func(t *testing.T) {
resp, err := fetchEmbed(
"https://www.youtube.com/watch?v=8jPQjjsBbIc",
makeCandidate(Provider{
&Provider{
Name: "YouTube",
URL: "https://www.youtube.com/",
Endpoints: []Endpoint{Endpoint{
Endpoints: []Endpoint{{
Schemes: []string{
"https://*.youtube.com/watch*",
"https://*.youtube.com/v/*\"",
@ -23,7 +23,7 @@ func TestFetchEmbed(t *testing.T) {
URL: "https://www.youtube.com/oembed",
Discovery: true,
}},
}),
},
&Params{
MaxWidth: 250,
MaxHeight: 250,
@ -34,7 +34,6 @@ func TestFetchEmbed(t *testing.T) {
})
t.Run("invalid", func(t *testing.T) {
for _, url := range []string{
"",
"htt:/abc.com/failed-none-sense",
"https://abc.com/failed-none-sense",
"http://badcom/146753785",
@ -45,12 +44,12 @@ func TestFetchEmbed(t *testing.T) {
} {
url := url
t.Run(url, func(t *testing.T) {
c := findProvider(url)
if c == nil {
c = new(providerCandidate)
provider := findProvider(url)
if provider == nil {
provider = &Provider{Endpoints: []Endpoint{Endpoint{}}}
}
_, err := fetchEmbed(url, *c, nil)
_, err := fetchEmbed(url, provider, nil)
assert.Error(err)
})
}

View File

@ -44,26 +44,34 @@ func makeCandidate(p Provider) providerCandidate {
}
func findProvider(url string) *providerCandidate {
var candidates []providerCandidate
for _, provider := range providersList {
func findProvider(url string) *Provider {
var candidates []Provider
for _, provider := range Providers {
provider := provider
candidate := makeCandidate(provider)
if len(candidate.Schemes) == 0 {
if !strings.Contains(url, candidate.Domain) {
endpoint := provider.Endpoints[0]
domain := getHostname(endpoint.URL)
if domain != "" {
domain = strings.TrimPrefix(domain, "www.")
} else {
domain = ""
}
if len(endpoint.Schemes) == 0 {
if !strings.Contains(url, domain) {
continue
}
candidates = append(candidates, candidate)
candidates = append(candidates, provider)
continue
}
for _, scheme := range candidate.Schemes {
for _, scheme := range endpoint.Schemes {
scheme := scheme
reg := regexp.MustCompile(strings.Replace(scheme, "*", "(.*)", -1))
if !reg.MatchString(url) {
continue
}
candidates = append(candidates, candidate)
candidates = append(candidates, provider)
break
}
}

View File

@ -17,6 +17,7 @@ func TestGetHostname(t *testing.T) {
"http://": "",
"": "",
} {
k, v := k, v
t.Run(k, func(t *testing.T) { assert.Equal(v, getHostname(k)) })
}
}
@ -26,7 +27,7 @@ func TestMakeCandidate(t *testing.T) {
Name: "YouTube",
URL: "https://www.youtube.com/",
Endpoints: []Endpoint{
Endpoint{
{
Schemes: []string{
"https://*.youtube.com/watch*",
"https://*.youtube.com/v/*\"",

View File

@ -1,24 +1,35 @@
package oembed
import (
"golang.org/x/xerrors"
)
import "golang.org/x/xerrors"
var (
ErrInvalidInputURL = xerrors.New("invalid input url")
ErrNoProviderFound = xerrors.New("no provider found with given url")
)
func Extract(url string, params *Params) (*Response, error) {
// Extract try fetch oEmbed object for input url with params (if represent).
// Return OEmbed if success.
func Extract(url string, params *Params) (*OEmbed, error) {
if !isValidURL(url) {
return nil, ErrInvalidInputURL
return nil, Error{
Message: "invalid input url",
URL: url,
}
}
if c := findProvider(url); c != nil {
return fetchEmbed(url, *c, params)
if provider := findProvider(url); provider != nil {
resp, err := fetchEmbed(url, provider, params)
if err != nil {
return nil, Error{
Message: err.Error(),
URL: url,
Details: xerrors.Caller(1),
}
}
return resp, nil
}
return nil, Error{
Message: "no provider found with given url",
URL: url,
}
return nil, ErrNoProviderFound
}
// HasProvider checks what input url has oEmbed provider
func HasProvider(url string) bool {
return findProvider(url) != nil
}

44
sync.go
View File

@ -1,46 +1,42 @@
package oembed
import (
"io/ioutil"
"log"
"os"
"path/filepath"
"time"
"github.com/gobuffalo/packr"
json "github.com/pquerna/ffjson/ffjson"
http "github.com/valyala/fasthttp"
)
const source string = "https://oembed.com/providers.json"
// SourceURL is a official url of supported providers list
const SourceURL string = "https://oembed.com/providers.json"
var (
providersList []Provider
// Providers contains all default (or new synced) providers
var Providers []Provider //nolint:gochecknoglobals
target = filepath.Join(
os.Getenv("GOPATH"), "src", "gitlab.com", "toby3d", "oembed", "assets", "providers.json",
)
)
func init() {
if err := fetch(source, target); err != nil {
func init() { //nolint:gochecknoinits
if err := Sync(SourceURL); err != nil {
panic(err)
}
}
func fetch(url, target string) error {
status, src, err := http.Get(nil, url)
// Sync try update Providers variable via request and parsing body of sourceURL
func Sync(sourceURL string) error {
status, src, err := http.GetTimeout(nil, sourceURL, 2*time.Second)
if err != nil || status != http.StatusOK {
if src, err = ioutil.ReadFile(target); err != nil {
return err
if src, err = packr.NewBox("./assets").Find("providers.json"); err != nil {
return Error{
Message: err.Error(),
URL: sourceURL,
}
}
}
if err := json.Unmarshal(src, &providersList); err != nil {
return err
}
if status == http.StatusOK {
if err = ioutil.WriteFile(target, src, os.ModePerm); err != nil {
return err
if err = json.Unmarshal(src, &Providers); err != nil {
return Error{
Message: err.Error(),
URL: sourceURL,
}
}

24
sync_test.go Normal file
View File

@ -0,0 +1,24 @@
package oembed
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSync(t *testing.T) {
t.Run("invalid", func(t *testing.T) {
t.Run("source url", func(t *testing.T) {
assert.NoError(t, Sync("wtf"))
assert.NotZero(t, len(Providers))
})
t.Run("resource body", func(t *testing.T) {
assert.Error(t, Sync("https://ddg.gg/"))
assert.NotZero(t, len(Providers))
})
})
t.Run("valid url", func(t *testing.T) {
assert.NoError(t, Sync(SourceURL))
assert.NotZero(t, len(Providers))
})
}

View File

@ -2,12 +2,14 @@
package oembed
type (
// Provider represent a single provider info
Provider struct {
Name string `json:"provider_name"`
URL string `json:"provider_url"`
Endpoints []Endpoint `json:"endpoints"`
}
// Provider represent a single endpoint of Provider
Endpoint struct {
Schemes []string `json:"schemes,omitempty"`
URL string `json:"url"`
@ -17,7 +19,7 @@ type (
// Response can specify a resource type, such as photo or video.
// Each type has specific parameters associated with it.
Response struct {
OEmbed struct {
// The resource type.
Type string `json:"type"` // required
@ -88,7 +90,7 @@ type (
// Link type allow a provider to return any generic embed data (such as title and author_name), without
// providing either the url or html parameters. The consumer may then link to the resource, using the URL
// specified in the original request.
Link string
// Link string
// Rich is used for rich HTML content that does not fall under one of the other categories.
Rich struct {

File diff suppressed because it is too large Load Diff