parent
dcf9e3c2ca
commit
531b14524c
GPG Key ID:
1F14E25B7C119FC5
13
go.mod
13
go.mod
|
|
@ -4,11 +4,11 @@ go 1.18
|
|||
|
||||
require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0
|
||||
github.com/brianvoe/gofakeit/v6 v6.15.0
|
||||
github.com/brianvoe/gofakeit/v6 v6.16.0
|
||||
github.com/fasthttp/router v1.4.10
|
||||
github.com/goccy/go-json v0.9.7
|
||||
github.com/jmoiron/sqlx v1.3.5
|
||||
github.com/lestrrat-go/jwx v1.2.25
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.2
|
||||
github.com/spf13/viper v1.12.0
|
||||
github.com/stretchr/testify v1.7.2
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||
|
|
@ -21,7 +21,7 @@ require (
|
|||
inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
|
||||
modernc.org/sqlite v1.17.3
|
||||
source.toby3d.me/toby3d/form v0.3.0
|
||||
source.toby3d.me/toby3d/middleware v0.9.1
|
||||
source.toby3d.me/toby3d/middleware v0.9.2
|
||||
willnorris.com/go/microformats v1.1.1
|
||||
)
|
||||
|
||||
|
|
@ -36,9 +36,9 @@ require (
|
|||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.6 // indirect
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httprc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/iter v1.0.2 // indirect
|
||||
github.com/lestrrat-go/option v1.0.0 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
|
|
@ -47,7 +47,6 @@ require (
|
|||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.2 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/savsgio/dictpool v0.0.0-20220406081701-03de5edb2e6d // indirect
|
||||
|
|
@ -62,10 +61,10 @@ require (
|
|||
go4.org/intern v0.0.0-20220301175310-a089fc204883 // indirect
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
|
||||
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
|
||||
golang.org/x/tools v0.1.10 // indirect
|
||||
golang.org/x/tools v0.1.11 // indirect
|
||||
gopkg.in/ini.v1 v1.66.6 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
|
|
|||
28
go.sum
28
go.sum
|
|
@ -47,8 +47,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY
|
|||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
|
||||
github.com/brianvoe/gofakeit/v6 v6.15.0 h1:lJPGJZ2/07TRGDazyTzD5b18N3y4tmmJpdhCUw18FlI=
|
||||
github.com/brianvoe/gofakeit/v6 v6.15.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
|
||||
github.com/brianvoe/gofakeit/v6 v6.16.0 h1:EelCqtfArd8ppJ0z+TpOxXH8sVWNPBadPNdCDSMMw7k=
|
||||
github.com/brianvoe/gofakeit/v6 v6.16.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
|
|
@ -62,7 +62,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
|
|
@ -185,18 +184,16 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
|||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
|
||||
github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
|
||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
||||
github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
|
||||
github.com/lestrrat-go/httprc v1.0.1 h1:Cnc4NxIySph38pQPzKbjg5OkKsGR/Cf5xcWt5OlSUDI=
|
||||
github.com/lestrrat-go/httprc v1.0.1/go.mod h1:5Ml+nB++j6IC0e6LzefJnrpMQDKgDwDCaIQQzhbqhJM=
|
||||
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
|
||||
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
|
||||
github.com/lestrrat-go/jwx v1.2.25 h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA=
|
||||
github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.2 h1:wkq9jwCkF3xrykISzn0Eksd7NEMOZ9yvCdnEpovIJX8=
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.2/go.mod h1:xV8+xRcrKbmnScV8adOzUuuTrL8aAZJoY4q2JAqIYU8=
|
||||
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
|
||||
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
|
|
@ -230,7 +227,6 @@ github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS
|
|||
github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI=
|
||||
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
|
||||
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
|
@ -342,8 +338,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -518,8 +514,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
|
|||
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
|
||||
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
|
||||
golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=
|
||||
golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -685,7 +681,7 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
|||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
source.toby3d.me/toby3d/form v0.3.0 h1:kI8apdFeVr+koqTTGVoIRiR5NMqjrhCJlajYlu+1bVw=
|
||||
source.toby3d.me/toby3d/form v0.3.0/go.mod h1:drlHMC+j/gb5zsttCSwx8qcYsbaRW+wFfE8bK6y+oeY=
|
||||
source.toby3d.me/toby3d/middleware v0.9.1 h1:MjXDs57SbrIndEVFJHN36ZiO2OgnXNSajrLqa87vzZQ=
|
||||
source.toby3d.me/toby3d/middleware v0.9.1/go.mod h1:Z67Z26wyW5USWnAYBYiA6mLubyzsHkbY6sWuIzs4304=
|
||||
source.toby3d.me/toby3d/middleware v0.9.2 h1:HInjjZaN7GTqlWq32XscJs4Wf0taFG6OhyTAJrED1vA=
|
||||
source.toby3d.me/toby3d/middleware v0.9.2/go.mod h1:MWedNnEpLCOk2rgjlfjpkn38t+1j53htSrp3lf6pC34=
|
||||
willnorris.com/go/microformats v1.1.1 h1:h5tk2luq6KBIRcwMGdksxdeea4GGuWrRFie5460OAbo=
|
||||
willnorris.com/go/microformats v1.1.1/go.mod h1:kvVnWrkkEscVAIITCEoiTX66Hcyg59C7q0E49mb9TJ0=
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ func initLookup() {
|
|||
addWeightedLookup()
|
||||
addMinecraftLookup()
|
||||
addCelebrityLookup()
|
||||
addDatabaseSQLLookup()
|
||||
}
|
||||
|
||||
// NewMapParams will create a new MapParams
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
package gofakeit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SQLOptions struct {
|
||||
Table string `json:"table" xml:"table"` // Table name we are inserting into
|
||||
Count int `json:"count" xml:"count"` // How many entries (tuples) we're generating
|
||||
Fields []Field `json:"fields" xml:"fields"` // The fields to be generated
|
||||
}
|
||||
|
||||
func SQL(so *SQLOptions) (string, error) { return sqlFunc(globalFaker.Rand, so) }
|
||||
|
||||
func (f *Faker) SQL(so *SQLOptions) (string, error) { return sqlFunc(f.Rand, so) }
|
||||
|
||||
func sqlFunc(r *rand.Rand, so *SQLOptions) (string, error) {
|
||||
if so.Table == "" {
|
||||
return "", errors.New("must provide table name to generate SQL")
|
||||
}
|
||||
if so.Fields == nil || len(so.Fields) <= 0 {
|
||||
return "", errors.New(("must pass fields in order to generate SQL queries"))
|
||||
}
|
||||
if so.Count <= 0 {
|
||||
return "", errors.New("must have entry count")
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("INSERT INTO " + so.Table + " ")
|
||||
|
||||
// Loop through each field and put together column names
|
||||
var cols []string
|
||||
for _, f := range so.Fields {
|
||||
cols = append(cols, f.Name)
|
||||
}
|
||||
sb.WriteString("(" + strings.Join(cols, ", ") + ")")
|
||||
|
||||
sb.WriteString(" VALUES ")
|
||||
for i := 0; i < so.Count; i++ {
|
||||
// Start opening value
|
||||
sb.WriteString("(")
|
||||
|
||||
// Now, we need to add all of our fields
|
||||
var endStr string
|
||||
for ii, field := range so.Fields {
|
||||
// Set end of value string
|
||||
endStr = ", "
|
||||
if ii == len(so.Fields)-1 {
|
||||
endStr = ""
|
||||
}
|
||||
|
||||
// If autoincrement, add based upon loop
|
||||
if field.Function == "autoincrement" {
|
||||
sb.WriteString(fmt.Sprintf("%d%s", i+1, endStr))
|
||||
continue
|
||||
}
|
||||
|
||||
// Get the function info for the field
|
||||
funcInfo := GetFuncLookup(field.Function)
|
||||
if funcInfo == nil {
|
||||
return "", errors.New("invalid function, " + field.Function + " does not exist")
|
||||
}
|
||||
|
||||
// Generate the value
|
||||
val, err := funcInfo.Generate(r, &field.Params, funcInfo)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Convert the output value to the proper SQL type
|
||||
convertType := sqlConvertType(funcInfo.Output, val)
|
||||
|
||||
// If its the last field, we need to close the value
|
||||
sb.WriteString(convertType + endStr)
|
||||
}
|
||||
|
||||
// If its the last value, we need to close the value
|
||||
if i == so.Count-1 {
|
||||
sb.WriteString(");")
|
||||
} else {
|
||||
sb.WriteString("),")
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// sqlConvertType will take in a type and value and convert it to the proper SQL type
|
||||
func sqlConvertType(t string, val interface{}) string {
|
||||
switch t {
|
||||
case "string":
|
||||
return `'` + fmt.Sprintf("%v", val) + `'`
|
||||
case "[]byte":
|
||||
return `'` + fmt.Sprintf("%s", val) + `'`
|
||||
default:
|
||||
return fmt.Sprintf("%v", val)
|
||||
}
|
||||
}
|
||||
|
||||
func addDatabaseSQLLookup() {
|
||||
AddFuncLookup("sql", Info{
|
||||
Display: "SQL",
|
||||
Category: "database",
|
||||
Description: "Generates an object or an array of objects in json format",
|
||||
Example: `INSERT INTO people
|
||||
(id, first_name, price, age, created_at)
|
||||
VALUES
|
||||
(1, 'Markus', 804.92, 21, '1937-01-30 07:58:01'),
|
||||
(2, 'Santino', 235.13, 40, '1964-07-07 22:25:40');`,
|
||||
Output: "string",
|
||||
ContentType: "application/sql",
|
||||
Params: []Param{
|
||||
{Field: "table", Display: "Table", Type: "string", Description: "Name of the table to insert into"},
|
||||
{Field: "count", Display: "Count", Type: "int", Default: "100", Description: "Number of inserts to generate"},
|
||||
{Field: "fields", Display: "Fields", Type: "[]Field", Description: "Fields containing key name and function to run in json format"},
|
||||
},
|
||||
Generate: func(r *rand.Rand, m *MapParams, info *Info) (interface{}, error) {
|
||||
so := SQLOptions{}
|
||||
|
||||
table, err := info.GetString(m, "table")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
so.Table = table
|
||||
|
||||
count, err := info.GetInt(m, "count")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
so.Count = count
|
||||
|
||||
fieldsStr, err := info.GetStringArray(m, "fields")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check to make sure fields has length
|
||||
if len(fieldsStr) > 0 {
|
||||
so.Fields = make([]Field, len(fieldsStr))
|
||||
|
||||
for i, f := range fieldsStr {
|
||||
// Unmarshal fields string into fields array
|
||||
err = json.Unmarshal([]byte(f), &so.Fields[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sqlFunc(r, &so)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
issues:
|
||||
exclude-rules:
|
||||
- path: /*_example_test.go
|
||||
linters:
|
||||
- errcheck
|
||||
- forbidigo
|
||||
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
v2.0.8 - 28 Feb 2021
|
||||
* Fix possible goroutine leak (#30)
|
||||
|
||||
v2.0.7 - 26 Jan 2021
|
||||
* Cosmetic go.mod / go.sum changes
|
||||
|
||||
v2.0.6 - 25 Jan 2021
|
||||
* Add jitter to constant backoff
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
# backoff  [](https://pkg.go.dev/github.com/lestrrat-go/backoff/v2)
|
||||
|
||||
Idiomatic backoff for Go
|
||||
|
||||
This library is an implementation of backoff algorithm for retrying operations
|
||||
in an idiomatic Go way. It respects `context.Context` natively, and the critical
|
||||
notifications are done through *channel operations*, allowing you to write code
|
||||
that is both more explicit and flexibile.
|
||||
|
||||
For a longer discussion, [please read this article](https://medium.com/@lestrrat/yak-shaving-with-backoff-libraries-in-go-80240f0aa30c)
|
||||
|
||||
# IMPORT
|
||||
|
||||
```go
|
||||
import "github.com/lestrrat-go/backoff/v2"
|
||||
```
|
||||
|
||||
# SYNOPSIS
|
||||
|
||||
```go
|
||||
func ExampleExponential() {
|
||||
p := backoff.Exponential(
|
||||
backoff.WithMinInterval(time.Second),
|
||||
backoff.WithMaxInterval(time.Minute),
|
||||
backoff.WithJitterFactor(0.05),
|
||||
)
|
||||
|
||||
flakyFunc := func(a int) (int, error) {
|
||||
// silly function that only succeeds if the current call count is
|
||||
// divisible by either 3 or 5 but not both
|
||||
switch {
|
||||
case a%15 == 0:
|
||||
return 0, errors.New(`invalid`)
|
||||
case a%3 == 0 || a%5 == 0:
|
||||
return a, nil
|
||||
}
|
||||
return 0, errors.New(`invalid`)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
retryFunc := func(v int) (int, error) {
|
||||
b := p.Start(ctx)
|
||||
for backoff.Continue(b) {
|
||||
x, err := flakyFunc(v)
|
||||
if err == nil {
|
||||
return x, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New(`failed to get value`)
|
||||
}
|
||||
|
||||
retryFunc(15)
|
||||
}
|
||||
```
|
||||
|
||||
# POLICIES
|
||||
|
||||
Policy objects describe a backoff policy, and are factories to create backoff Controller objects.
|
||||
Controller objects does the actual coordination.
|
||||
Create a new controller for each invocation of a backoff enabled operation.
|
||||
This way the controller object is protected from concurrent access (if you have one) and can easily be discarded
|
||||
|
||||
## Null
|
||||
|
||||
A null policy means there's no backoff.
|
||||
|
||||
For example, if you were to support both using and not using a backoff in your code you can say
|
||||
|
||||
```go
|
||||
var p backoff.Policy
|
||||
if useBackoff {
|
||||
p = backoff.Exponential(...)
|
||||
} else {
|
||||
p = backoff.Null()
|
||||
}
|
||||
c := p.Start(ctx)
|
||||
for backoff.Continue(c) {
|
||||
if err := doSomething(); err != nil {
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
```
|
||||
|
||||
Instead of
|
||||
|
||||
```go
|
||||
if useBackoff {
|
||||
p := backoff.Exponential(...)
|
||||
c := p.Start(ctx)
|
||||
for backoff.Continue(c) {
|
||||
if err := doSomething(); err != nil {
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if err := doSomething(); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Constant
|
||||
|
||||
A constant policy implements are backoff where the intervals are always the same
|
||||
|
||||
## Exponential
|
||||
|
||||
This is the most "common" of the backoffs. Intervals between calls are spaced out such that as you keep retrying, the intervals keep increasing.
|
||||
|
||||
# FAQ
|
||||
|
||||
## I'm getting "package github.com/lestrrat-go/backoff/v2: no Go files in /go/src/github.com/lestrrat-go/backoff/v2"
|
||||
|
||||
You are using Go in GOPATH mode, which was the way before [Go Modules](https://blog.golang.org/using-go-modules) were introduced in Go 1.11 (Aug 2018).
|
||||
GOPATH has slowly been phased out, and in Go 1.14 onwards, Go Modules pretty much Just Work.
|
||||
Go 1.16 introduced more visible changes that forces users to be aware of the existance of go.mod files.
|
||||
|
||||
The short answer when you you get the above error is: **Migrate to using Go Modules**.
|
||||
This is simple: All you need to do is to include a go.mod (and go.sum) file to your library or app.
|
||||
|
||||
For example, if you have previously been doing this:
|
||||
|
||||
```
|
||||
git clone git@github.com:myusername/myawesomeproject.git
|
||||
cd myawesomeproject
|
||||
go get ./...
|
||||
```
|
||||
|
||||
First include go.mod and go.sum in your repository:
|
||||
|
||||
```
|
||||
git clone git@github.com:myusername/myawesomeproject.git
|
||||
cd myawesomeproject
|
||||
go mod init
|
||||
go mod tidy
|
||||
git add go.mod go.sum
|
||||
git commit -m "Add go.mod and go.sum" -a
|
||||
git push
|
||||
```
|
||||
|
||||
Then from subsequent calls:
|
||||
|
||||
```
|
||||
git clone git@github.com:myusername/myawesomeproject.git
|
||||
cd myawesomeproject
|
||||
go build # or go test, or go run, or whatever.
|
||||
```
|
||||
|
||||
This will tell go to respect tags, and will automatically pick up the latest version of github.com/lestrrat-go/backoff
|
||||
|
||||
If you really can't do this, then the quick and dirty workaround is to just copy the files over to /v2 directory of this library
|
||||
|
||||
```
|
||||
BACKOFF=github.com/lestrrat-go/backoff
|
||||
go get github.com/lestrrat-go/backoff
|
||||
if [[ if ! -d "$GOPATH/src/$BACKOFF/v2" ]]; then
|
||||
mkdir "$GOPATH/src/$BACKOFF/v2" # just to make sure it exists
|
||||
fi
|
||||
cp "$GOPATH/src/$BACKOFF/*.go" "$GOPATH/src/$BACKOFF/v2"
|
||||
|
||||
git clone git@github.com:myusername/myawesomeproject.git
|
||||
cd myawesomeproject
|
||||
go get ./...
|
||||
```
|
||||
|
||||
## Why won't you just add the files in /v2?
|
||||
|
||||
Because it's hard to maintain multiple sources of truth. Sorry, I don't get paid to do this.
|
||||
I will not hold anything against you if you decided to fork this to your repository, and move files to your own /v2 directory.
|
||||
Then, if you have a go.mod in your app, you can just do
|
||||
|
||||
```
|
||||
go mod edit -replace github.com/lestrrat-go/backoff/v2=github.com/myusername/myawesomemfork/v2
|
||||
```
|
||||
|
||||
Oh, wait, then you already have go.mod, so this is a non-issue.
|
||||
|
||||
...Yeah, just migrate to using go modules, please?
|
||||
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package backoff
|
||||
|
||||
// Null creates a new NullPolicy object
|
||||
func Null() Policy {
|
||||
return NewNull()
|
||||
}
|
||||
|
||||
// Constant creates a new ConstantPolicy object
|
||||
func Constant(options ...Option) Policy {
|
||||
return NewConstantPolicy(options...)
|
||||
}
|
||||
|
||||
// Constant creates a new ExponentialPolicy object
|
||||
func Exponential(options ...ExponentialOption) Policy {
|
||||
return NewExponentialPolicy(options...)
|
||||
}
|
||||
|
||||
// Continue is a convenience function to check when we can fire
|
||||
// the next invocation of the desired backoff code
|
||||
//
|
||||
// for backoff.Continue(c) {
|
||||
// ... your code ...
|
||||
// }
|
||||
func Continue(c Controller) bool {
|
||||
select {
|
||||
case <-c.Done():
|
||||
return false
|
||||
case _, ok := <-c.Next():
|
||||
return ok
|
||||
}
|
||||
}
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ConstantInterval struct {
|
||||
interval time.Duration
|
||||
jitter jitter
|
||||
}
|
||||
|
||||
func NewConstantInterval(options ...ConstantOption) *ConstantInterval {
|
||||
jitterFactor := 0.0
|
||||
interval := time.Minute
|
||||
var rng Random
|
||||
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identInterval{}:
|
||||
interval = option.Value().(time.Duration)
|
||||
case identJitterFactor{}:
|
||||
jitterFactor = option.Value().(float64)
|
||||
case identRNG{}:
|
||||
rng = option.Value().(Random)
|
||||
}
|
||||
}
|
||||
|
||||
return &ConstantInterval{
|
||||
interval: interval,
|
||||
jitter: newJitter(jitterFactor, rng),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *ConstantInterval) Next() time.Duration {
|
||||
return time.Duration(g.jitter.apply(float64(g.interval)))
|
||||
}
|
||||
|
||||
type ConstantPolicy struct {
|
||||
cOptions []ControllerOption
|
||||
igOptions []ConstantOption
|
||||
}
|
||||
|
||||
func NewConstantPolicy(options ...Option) *ConstantPolicy {
|
||||
var cOptions []ControllerOption
|
||||
var igOptions []ConstantOption
|
||||
|
||||
for _, option := range options {
|
||||
switch opt := option.(type) {
|
||||
case ControllerOption:
|
||||
cOptions = append(cOptions, opt)
|
||||
default:
|
||||
igOptions = append(igOptions, opt.(ConstantOption))
|
||||
}
|
||||
}
|
||||
|
||||
return &ConstantPolicy{
|
||||
cOptions: cOptions,
|
||||
igOptions: igOptions,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ConstantPolicy) Start(ctx context.Context) Controller {
|
||||
ig := NewConstantInterval(p.igOptions...)
|
||||
return newController(ctx, ig, p.cOptions...)
|
||||
}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type controller struct {
|
||||
ctx context.Context
|
||||
cancel func()
|
||||
ig IntervalGenerator
|
||||
maxRetries int
|
||||
mu *sync.RWMutex
|
||||
next chan struct{} // user-facing channel
|
||||
resetTimer chan time.Duration
|
||||
retries int
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
func newController(ctx context.Context, ig IntervalGenerator, options ...ControllerOption) *controller {
|
||||
cctx, cancel := context.WithCancel(ctx) // DO NOT fire this cancel here
|
||||
|
||||
maxRetries := 10
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identMaxRetries{}:
|
||||
maxRetries = option.Value().(int)
|
||||
}
|
||||
}
|
||||
|
||||
c := &controller{
|
||||
cancel: cancel,
|
||||
ctx: cctx,
|
||||
ig: ig,
|
||||
maxRetries: maxRetries,
|
||||
mu: &sync.RWMutex{},
|
||||
next: make(chan struct{}, 1),
|
||||
resetTimer: make(chan time.Duration, 1),
|
||||
timer: time.NewTimer(ig.Next()),
|
||||
}
|
||||
|
||||
// enqueue a single fake event so the user gets to retry once
|
||||
c.next <- struct{}{}
|
||||
|
||||
go c.loop()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *controller) loop() {
|
||||
for {
|
||||
select {
|
||||
case <-c.ctx.Done():
|
||||
return
|
||||
case d := <-c.resetTimer:
|
||||
if !c.timer.Stop() {
|
||||
select {
|
||||
case <-c.timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
c.timer.Reset(d)
|
||||
case <-c.timer.C:
|
||||
select {
|
||||
case <-c.ctx.Done():
|
||||
return
|
||||
case c.next <- struct{}{}:
|
||||
}
|
||||
if c.maxRetries > 0 {
|
||||
c.retries++
|
||||
}
|
||||
|
||||
if !c.check() {
|
||||
c.cancel()
|
||||
return
|
||||
}
|
||||
c.resetTimer <- c.ig.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *controller) check() bool {
|
||||
if c.maxRetries > 0 && c.retries >= c.maxRetries {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *controller) Done() <-chan struct{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.ctx.Done()
|
||||
}
|
||||
|
||||
func (c *controller) Next() <-chan struct{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.next
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
// Package backoff implments backoff algorithms for retrying operations.
|
||||
//
|
||||
// Users first create an appropriate `Policy` object, and when the operation
|
||||
// that needs retrying is about to start, they kick the actual backoff
|
||||
//
|
||||
package backoff
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ExponentialInterval struct {
|
||||
current float64
|
||||
maxInterval float64
|
||||
minInterval float64
|
||||
multiplier float64
|
||||
jitter jitter
|
||||
}
|
||||
|
||||
const (
|
||||
defaultMaxInterval = float64(time.Minute)
|
||||
defaultMinInterval = float64(500 * time.Millisecond)
|
||||
defaultMultiplier = 1.5
|
||||
)
|
||||
|
||||
func NewExponentialInterval(options ...ExponentialOption) *ExponentialInterval {
|
||||
jitterFactor := 0.0
|
||||
maxInterval := defaultMaxInterval
|
||||
minInterval := defaultMinInterval
|
||||
multiplier := defaultMultiplier
|
||||
var rng Random
|
||||
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identJitterFactor{}:
|
||||
jitterFactor = option.Value().(float64)
|
||||
case identMaxInterval{}:
|
||||
maxInterval = float64(option.Value().(time.Duration))
|
||||
case identMinInterval{}:
|
||||
minInterval = float64(option.Value().(time.Duration))
|
||||
case identMultiplier{}:
|
||||
multiplier = option.Value().(float64)
|
||||
case identRNG{}:
|
||||
rng = option.Value().(Random)
|
||||
}
|
||||
}
|
||||
|
||||
if minInterval > maxInterval {
|
||||
minInterval = maxInterval
|
||||
}
|
||||
if multiplier <= 1 {
|
||||
multiplier = defaultMultiplier
|
||||
}
|
||||
|
||||
return &ExponentialInterval{
|
||||
maxInterval: maxInterval,
|
||||
minInterval: minInterval,
|
||||
multiplier: multiplier,
|
||||
jitter: newJitter(jitterFactor, rng),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *ExponentialInterval) Next() time.Duration {
|
||||
var next float64
|
||||
if g.current == 0 {
|
||||
next = g.minInterval
|
||||
} else {
|
||||
next = g.current * g.multiplier
|
||||
}
|
||||
|
||||
if next > g.maxInterval {
|
||||
next = g.maxInterval
|
||||
}
|
||||
if next < g.minInterval {
|
||||
next = g.minInterval
|
||||
}
|
||||
|
||||
// Apply jitter *AFTER* we calculate the base interval
|
||||
next = g.jitter.apply(next)
|
||||
g.current = next
|
||||
return time.Duration(next)
|
||||
}
|
||||
|
||||
type ExponentialPolicy struct {
|
||||
cOptions []ControllerOption
|
||||
igOptions []ExponentialOption
|
||||
}
|
||||
|
||||
func NewExponentialPolicy(options ...ExponentialOption) *ExponentialPolicy {
|
||||
var cOptions []ControllerOption
|
||||
var igOptions []ExponentialOption
|
||||
|
||||
for _, option := range options {
|
||||
switch opt := option.(type) {
|
||||
case ControllerOption:
|
||||
cOptions = append(cOptions, opt)
|
||||
default:
|
||||
igOptions = append(igOptions, opt)
|
||||
}
|
||||
}
|
||||
|
||||
return &ExponentialPolicy{
|
||||
cOptions: cOptions,
|
||||
igOptions: igOptions,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ExponentialPolicy) Start(ctx context.Context) Controller {
|
||||
ig := NewExponentialInterval(p.igOptions...)
|
||||
return newController(ctx, ig, p.cOptions...)
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/option"
|
||||
)
|
||||
|
||||
type Option = option.Interface
|
||||
|
||||
type Controller interface {
|
||||
Done() <-chan struct{}
|
||||
Next() <-chan struct{}
|
||||
}
|
||||
|
||||
type IntervalGenerator interface {
|
||||
Next() time.Duration
|
||||
}
|
||||
|
||||
// Policy is an interface for the backoff policies that this package
|
||||
// implements. Users must create a controller object from this
|
||||
// policy to actually do anything with it
|
||||
type Policy interface {
|
||||
Start(context.Context) Controller
|
||||
}
|
||||
|
||||
type Random interface {
|
||||
Float64() float64
|
||||
}
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
)
|
||||
|
||||
type jitter interface {
|
||||
apply(interval float64) float64
|
||||
}
|
||||
|
||||
func newJitter(jitterFactor float64, rng Random) jitter {
|
||||
if jitterFactor <= 0 || jitterFactor >= 1 {
|
||||
return newNopJitter()
|
||||
}
|
||||
return newRandomJitter(jitterFactor, rng)
|
||||
}
|
||||
|
||||
type nopJitter struct{}
|
||||
|
||||
func newNopJitter() *nopJitter {
|
||||
return &nopJitter{}
|
||||
}
|
||||
|
||||
func (j *nopJitter) apply(interval float64) float64 {
|
||||
return interval
|
||||
}
|
||||
|
||||
type randomJitter struct {
|
||||
jitterFactor float64
|
||||
rng Random
|
||||
}
|
||||
|
||||
func newRandomJitter(jitterFactor float64, rng Random) *randomJitter {
|
||||
if rng == nil {
|
||||
// if we have a jitter factor, and no RNG is provided, create one.
|
||||
// This is definitely not "secure", but well, if you care enough,
|
||||
// you would provide one
|
||||
rng = rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
}
|
||||
|
||||
return &randomJitter{
|
||||
jitterFactor: jitterFactor,
|
||||
rng: rng,
|
||||
}
|
||||
}
|
||||
|
||||
func (j *randomJitter) apply(interval float64) float64 {
|
||||
jitterDelta := interval * j.jitterFactor
|
||||
jitterMin := interval - jitterDelta
|
||||
jitterMax := interval + jitterDelta
|
||||
|
||||
// Get a random value from the range [minInterval, maxInterval].
|
||||
// The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then
|
||||
// we want a 33% chance for selecting either 1, 2 or 3.
|
||||
//
|
||||
// see also: https://github.com/cenkalti/backoff/blob/c2975ffa541a1caeca5f76c396cb8c3e7b3bb5f8/exponential.go#L154-L157
|
||||
return jitterMin + j.rng.Float64()*(jitterMax-jitterMin+1)
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// NullPolicy does not do any backoff. It allows the caller
|
||||
// to execute the desired code once, and no more
|
||||
type NullPolicy struct{}
|
||||
|
||||
func NewNull() *NullPolicy {
|
||||
return &NullPolicy{}
|
||||
}
|
||||
|
||||
func (p *NullPolicy) Start(ctx context.Context) Controller {
|
||||
return newNullController(ctx)
|
||||
}
|
||||
|
||||
type nullController struct {
|
||||
mu *sync.RWMutex
|
||||
ctx context.Context
|
||||
next chan struct{}
|
||||
}
|
||||
|
||||
func newNullController(ctx context.Context) *nullController {
|
||||
cctx, cancel := context.WithCancel(ctx)
|
||||
c := &nullController{
|
||||
mu: &sync.RWMutex{},
|
||||
ctx: cctx,
|
||||
next: make(chan struct{}), // NO BUFFER
|
||||
}
|
||||
go func(ch chan struct{}, cancel func()) {
|
||||
ch <- struct{}{}
|
||||
close(ch)
|
||||
cancel()
|
||||
}(c.next, cancel)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *nullController) Done() <-chan struct{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.ctx.Done()
|
||||
}
|
||||
|
||||
func (c *nullController) Next() <-chan struct{} {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.next
|
||||
}
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
package backoff
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/option"
|
||||
)
|
||||
|
||||
type identInterval struct{}
|
||||
type identJitterFactor struct{}
|
||||
type identMaxInterval struct{}
|
||||
type identMaxRetries struct{}
|
||||
type identMinInterval struct{}
|
||||
type identMultiplier struct{}
|
||||
type identRNG struct{}
|
||||
|
||||
// ControllerOption is an option that may be passed to Policy objects,
|
||||
// but are ultimately passed down to the Controller objects.
|
||||
// (Normally you do not have to care about the distinction)
|
||||
type ControllerOption interface {
|
||||
ConstantOption
|
||||
ExponentialOption
|
||||
CommonOption
|
||||
controllerOption()
|
||||
}
|
||||
|
||||
type controllerOption struct {
|
||||
Option
|
||||
}
|
||||
|
||||
func (*controllerOption) exponentialOption() {}
|
||||
func (*controllerOption) controllerOption() {}
|
||||
func (*controllerOption) constantOption() {}
|
||||
|
||||
// ConstantOption is an option that is used by the Constant policy.
|
||||
type ConstantOption interface {
|
||||
Option
|
||||
constantOption()
|
||||
}
|
||||
|
||||
type constantOption struct {
|
||||
Option
|
||||
}
|
||||
|
||||
func (*constantOption) constantOption() {}
|
||||
|
||||
// ExponentialOption is an option that is used by the Exponential policy.
|
||||
type ExponentialOption interface {
|
||||
Option
|
||||
exponentialOption()
|
||||
}
|
||||
|
||||
type exponentialOption struct {
|
||||
Option
|
||||
}
|
||||
|
||||
func (*exponentialOption) exponentialOption() {}
|
||||
|
||||
// CommonOption is an option that can be passed to any of the backoff policies.
|
||||
type CommonOption interface {
|
||||
ExponentialOption
|
||||
ConstantOption
|
||||
}
|
||||
|
||||
type commonOption struct {
|
||||
Option
|
||||
}
|
||||
|
||||
func (*commonOption) constantOption() {}
|
||||
func (*commonOption) exponentialOption() {}
|
||||
|
||||
// WithMaxRetries specifies the maximum number of attempts that can be made
|
||||
// by the backoff policies. By default each policy tries up to 10 times.
|
||||
//
|
||||
// If you would like to retry forever, specify "0" and pass to the constructor
|
||||
// of each policy.
|
||||
//
|
||||
// This option can be passed to all policy constructors except for NullPolicy
|
||||
func WithMaxRetries(v int) ControllerOption {
|
||||
return &controllerOption{option.New(identMaxRetries{}, v)}
|
||||
}
|
||||
|
||||
// WithInterval specifies the constant interval used in ConstantPolicy and
|
||||
// ConstantInterval.
|
||||
// The default value is 1 minute.
|
||||
func WithInterval(v time.Duration) ConstantOption {
|
||||
return &constantOption{option.New(identInterval{}, v)}
|
||||
}
|
||||
|
||||
// WithMaxInterval specifies the maximum duration used in exponential backoff
|
||||
// The default value is 1 minute.
|
||||
func WithMaxInterval(v time.Duration) ExponentialOption {
|
||||
return &exponentialOption{option.New(identMaxInterval{}, v)}
|
||||
}
|
||||
|
||||
// WithMinInterval specifies the minimum duration used in exponential backoff.
|
||||
// The default value is 500ms.
|
||||
func WithMinInterval(v time.Duration) ExponentialOption {
|
||||
return &exponentialOption{option.New(identMinInterval{}, v)}
|
||||
}
|
||||
|
||||
// WithMultiplier specifies the factor in which the backoff intervals are
|
||||
// increased. By default this value is set to 1.5, which means that for
|
||||
// every iteration a 50% increase in the interval for every iteration
|
||||
// (up to the value specified by WithMaxInterval). this value must be greater
|
||||
// than 1.0. If the value is less than equal to 1.0, the default value
|
||||
// of 1.5 is used.
|
||||
func WithMultiplier(v float64) ExponentialOption {
|
||||
return &exponentialOption{option.New(identMultiplier{}, v)}
|
||||
}
|
||||
|
||||
// WithJitterFactor enables some randomness (jittering) in the computation of
|
||||
// the backoff intervals. This value must be between 0.0 < v < 1.0. If a
|
||||
// value outside of this range is specified, the value will be silently
|
||||
// ignored and jittering is disabled.
|
||||
//
|
||||
// This option can be passed to ExponentialPolicy or ConstantPolicy constructor
|
||||
func WithJitterFactor(v float64) CommonOption {
|
||||
return &commonOption{option.New(identJitterFactor{}, v)}
|
||||
}
|
||||
|
||||
// WithRNG specifies the random number generator used for jittering.
|
||||
// If not provided one will be created, but if you want a truly random
|
||||
// jittering, make sure to provide one that you explicitly initialized
|
||||
func WithRNG(v Random) CommonOption {
|
||||
return &commonOption{option.New(identRNG{}, v)}
|
||||
}
|
||||
|
|
@ -1,14 +1,15 @@
|
|||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, build with `go test -c`
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
|
||||
.glide/
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
run:
|
||||
|
||||
linters-settings:
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- shadow
|
||||
- fieldalignment
|
||||
|
||||
linters:
|
||||
enable-all: true
|
||||
disable:
|
||||
- cyclop
|
||||
- dupl
|
||||
- exhaustive
|
||||
- exhaustivestruct
|
||||
- errorlint
|
||||
- funlen
|
||||
- gci
|
||||
- gochecknoglobals
|
||||
- gochecknoinits
|
||||
- gocognit
|
||||
- gocritic
|
||||
- gocyclo
|
||||
- godot
|
||||
- godox
|
||||
- goerr113
|
||||
- gofumpt
|
||||
- golint #deprecated
|
||||
- gomnd
|
||||
- gosec
|
||||
- govet
|
||||
- interfacer # deprecated
|
||||
- ifshort
|
||||
- lll
|
||||
- maligned # deprecated
|
||||
- makezero
|
||||
- nakedret
|
||||
- nestif
|
||||
- nlreturn
|
||||
- paralleltest
|
||||
- scopelint # deprecated
|
||||
- tagliatelle
|
||||
- testpackage
|
||||
- thelper
|
||||
- wrapcheck
|
||||
- wsl
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
# not needed
|
||||
- path: /*.go
|
||||
text: "ST1003: should not use underscores in package names"
|
||||
linters:
|
||||
- stylecheck
|
||||
- path: /*.go
|
||||
text: "don't use an underscore in package name"
|
||||
linters:
|
||||
- revive
|
||||
- path: /main.go
|
||||
linters:
|
||||
- errcheck
|
||||
- path: internal/codegen/codegen.go
|
||||
linters:
|
||||
- errcheck
|
||||
- path: /*_test.go
|
||||
linters:
|
||||
- errcheck
|
||||
- forcetypeassert
|
||||
- path: /*_example_test.go
|
||||
linters:
|
||||
- forbidigo
|
||||
- path: cmd/jwx/jwx.go
|
||||
linters:
|
||||
- forbidigo
|
||||
|
||||
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
|
||||
max-issues-per-linter: 0
|
||||
|
||||
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
|
||||
max-same-issues: 0
|
||||
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
Changes
|
||||
=======
|
||||
|
||||
v1.0.1 29 Mar 2022
|
||||
* Bump dependency for github.com/lestrrat-go/httpcc to v1.0.1
|
||||
|
||||
v1.0.0 29 Mar 2022
|
||||
* Initial release, refactored out of github.com/lestrrat-go/jwx
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2018 lestrrat
|
||||
Copyright (c) 2022 lestrrat
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
# httprc
|
||||
|
||||
`httprc` is a HTTP "Refresh" Cache. Its aim is to cache a remote resource that
|
||||
can be fetched via HTTP, but keep the cached content up-to-date based on periodic
|
||||
refreshing.
|
||||
|
||||
# SYNOPSIS
|
||||
|
||||
<!-- INCLUDE(httprc_example_test.go) -->
|
||||
```go
|
||||
package httprc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/httprc"
|
||||
)
|
||||
|
||||
const (
|
||||
helloWorld = `Hello World!`
|
||||
goodbyeWorld = `Goodbye World!`
|
||||
)
|
||||
|
||||
func ExampleCache() {
|
||||
var mu sync.RWMutex
|
||||
|
||||
msg := helloWorld
|
||||
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set(`Cache-Control`, fmt.Sprintf(`max-age=%d`, 2))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
mu.RLock()
|
||||
fmt.Fprint(w, msg)
|
||||
mu.RUnlock()
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
errSink := httprc.ErrSinkFunc(func(err error) {
|
||||
fmt.Printf("%s\n", err)
|
||||
})
|
||||
|
||||
c := httprc.NewCache(ctx,
|
||||
httprc.WithErrSink(errSink),
|
||||
httprc.WithRefreshWindow(time.Second), // force checks every second
|
||||
)
|
||||
|
||||
c.Register(srv.URL,
|
||||
httprc.WithHTTPClient(srv.Client()), // we need client with TLS settings
|
||||
httprc.WithMinRefreshInterval(time.Second), // allow max-age=1 (smallest)
|
||||
)
|
||||
|
||||
payload, err := c.Get(ctx, srv.URL)
|
||||
if err != nil {
|
||||
fmt.Printf("%s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if string(payload.([]byte)) != helloWorld {
|
||||
fmt.Printf("payload mismatch: %s\n", payload)
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
msg = goodbyeWorld
|
||||
mu.Unlock()
|
||||
|
||||
time.Sleep(4 * time.Second)
|
||||
|
||||
payload, err = c.Get(ctx, srv.URL)
|
||||
if err != nil {
|
||||
fmt.Printf("%s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if string(payload.([]byte)) != goodbyeWorld {
|
||||
fmt.Printf("payload mismatch: %s\n", payload)
|
||||
return
|
||||
}
|
||||
|
||||
cancel()
|
||||
|
||||
// OUTPUT:
|
||||
}
|
||||
```
|
||||
source: [httprc_example_test.go](https://github.com/lestrrat-go/jwx/blob/main/httprc_example_test.go)
|
||||
<!-- END INCLUDE -->
|
||||
|
||||
# Sequence Diagram
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
autonumber
|
||||
actor User
|
||||
participant httprc.Cache
|
||||
participant httprc.Storage
|
||||
User->>httprc.Cache: Fetch URL `u`
|
||||
activate httprc.Storage
|
||||
httprc.Cache->>httprc.Storage: Fetch local cache for `u`
|
||||
alt Cache exists
|
||||
httprc.Storage-->httprc.Cache: Return local cache
|
||||
httprc.Cache-->>User: Return data
|
||||
Note over httprc.Storage: If the cache exists, there's nothing more to do.<br />The cached content will be updated periodically in httprc.Refresher
|
||||
deactivate httprc.Storage
|
||||
else Cache does not exist
|
||||
activate httprc.Fetcher
|
||||
httprc.Cache->>httprc.Fetcher: Fetch remote resource `u`
|
||||
httprc.Fetcher-->>httprc.Cache: Return fetched data
|
||||
deactivate httprc.Fetcher
|
||||
httprc.Cache-->>User: Return data
|
||||
httprc.Cache-)httprc.Refresher: Enqueue into auto-refresh queue
|
||||
activate httprc.Refresher
|
||||
loop Refresh Loop
|
||||
Note over httprc.Storage,httprc.Fetcher: Cached contents are updated synchronously
|
||||
httprc.Refresher->>httprc.Refresher: Wait until next refresh
|
||||
httprc.Refresher-->>httprc.Fetcher: Request fetch
|
||||
httprc.Fetcher->>httprc.Refresher: Return fetched data
|
||||
httprc.Refresher-->>httprc.Storage: Store new version in cache
|
||||
httprc.Refresher->>httprc.Refresher: Enqueue into auto-refresh queue (again)
|
||||
end
|
||||
deactivate httprc.Refresher
|
||||
end
|
||||
```
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrSink is an abstraction that allows users to consume errors
|
||||
// produced while the cache queue is running.
|
||||
type HTTPClient interface {
|
||||
Get(string) (*http.Response, error)
|
||||
}
|
||||
|
||||
// Cache represents a cache that stores resources locally, while
|
||||
// periodically refreshing the contents based on HTTP header values
|
||||
// and/or user-supplied hints.
|
||||
//
|
||||
// Refresh is performed _periodically_, and therefore the contents
|
||||
// are not kept up-to-date in real time. The interval between checks
|
||||
// for refreshes is called the refresh window.
|
||||
//
|
||||
// The default refresh window is 15 minutes. This means that if a
|
||||
// resource is fetched is at time T, and it is supposed to be
|
||||
// refreshed in 20 minutes, the next refresh for this resource will
|
||||
// happen at T+30 minutes (15+15 minutes).
|
||||
type Cache struct {
|
||||
mu sync.RWMutex
|
||||
queue *queue
|
||||
wl Whitelist
|
||||
}
|
||||
|
||||
const defaultRefreshWindow = 15 * time.Minute
|
||||
|
||||
// New creates a new Cache object.
|
||||
//
|
||||
// The context object in the argument controls the life-cycle of the
|
||||
// auto-refresh worker. If you cancel the `ctx`, then the automatic
|
||||
// refresh will stop working.
|
||||
//
|
||||
// Refresh will only be performed periodically where the interval between
|
||||
// refreshes are controlled by the `refresh window` variable. For example,
|
||||
// if the refresh window is every 5 minutes and the resource was queued
|
||||
// to be refreshed at 7 minutes, the resource will be refreshed after 10
|
||||
// minutes (in 2 refresh window time).
|
||||
//
|
||||
// The refresh window can be configured by using `httprc.WithRefreshWindow`
|
||||
// option. If you want refreshes to be performed more often, provide a smaller
|
||||
// refresh window. If you specify a refresh window that is smaller than 1
|
||||
// second, it will automatically be set to the default value, which is 15
|
||||
// minutes.
|
||||
//
|
||||
// Internally the HTTP fetching is done using a pool of HTTP fetch
|
||||
// workers. The default number of workers is 3. You may change this
|
||||
// number by specifying the `httprc.WithFetcherWorkerCount`
|
||||
func NewCache(ctx context.Context, options ...CacheOption) *Cache {
|
||||
var refreshWindow time.Duration
|
||||
var errSink ErrSink
|
||||
var wl Whitelist
|
||||
var fetcherOptions []FetcherOption
|
||||
for _, option := range options {
|
||||
//nolint:forcetypeassert
|
||||
switch option.Ident() {
|
||||
case identRefreshWindow{}:
|
||||
refreshWindow = option.Value().(time.Duration)
|
||||
case identFetcherWorkerCount{}, identWhitelist{}:
|
||||
fetcherOptions = append(fetcherOptions, option)
|
||||
case identErrSink{}:
|
||||
errSink = option.Value().(ErrSink)
|
||||
}
|
||||
}
|
||||
|
||||
if refreshWindow < time.Second {
|
||||
refreshWindow = defaultRefreshWindow
|
||||
}
|
||||
|
||||
fetch := NewFetcher(ctx, fetcherOptions...)
|
||||
queue := newQueue(ctx, refreshWindow, fetch, errSink)
|
||||
|
||||
return &Cache{
|
||||
queue: queue,
|
||||
wl: wl,
|
||||
}
|
||||
}
|
||||
|
||||
// Register configures a URL to be stored in the cache.
|
||||
//
|
||||
// For any given URL, the URL must be registered _BEFORE_ it is
|
||||
// accessed using `Get()` method.
|
||||
func (c *Cache) Register(u string, options ...RegisterOption) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if wl := c.wl; wl != nil {
|
||||
if !wl.IsAllowed(u) {
|
||||
return fmt.Errorf(`httprc.Cache: url %q has been rejected by whitelist`, u)
|
||||
}
|
||||
}
|
||||
|
||||
return c.queue.Register(u, options...)
|
||||
}
|
||||
|
||||
// Unregister removes the given URL `u` from the cache.
|
||||
//
|
||||
// Subsequent calls to `Get()` will fail until `u` is registered again.
|
||||
func (c *Cache) Unregister(u string) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.queue.Unregister(u)
|
||||
}
|
||||
|
||||
// IsRegistered returns true if the given URL `u` has already been
|
||||
// registered in the cache.
|
||||
func (c *Cache) IsRegistered(u string) bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.queue.IsRegistered(u)
|
||||
}
|
||||
|
||||
// Refresh is identical to Get(), except it always fetches the
|
||||
// specified resource anew, and updates the cached content
|
||||
func (c *Cache) Refresh(ctx context.Context, u string) (interface{}, error) {
|
||||
return c.getOrFetch(ctx, u, true)
|
||||
}
|
||||
|
||||
// Get returns the cached object.
|
||||
//
|
||||
// The context.Context argument is used to control the timeout for
|
||||
// synchronous fetches, when they need to happen. Synchronous fetches
|
||||
// will be performed when the cache does not contain the specified
|
||||
// resource.
|
||||
func (c *Cache) Get(ctx context.Context, u string) (interface{}, error) {
|
||||
return c.getOrFetch(ctx, u, false)
|
||||
}
|
||||
|
||||
func (c *Cache) getOrFetch(ctx context.Context, u string, forceRefresh bool) (interface{}, error) {
|
||||
c.mu.RLock()
|
||||
e, ok := c.queue.getRegistered(u)
|
||||
if !ok {
|
||||
c.mu.RUnlock()
|
||||
return nil, fmt.Errorf(`url %q is not registered (did you make sure to call Register() first?)`, u)
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
|
||||
// Only one goroutine may enter this section.
|
||||
e.acquireSem()
|
||||
|
||||
// has this entry been fetched? (but ignore and do a fetch
|
||||
// if forceRefresh is true)
|
||||
if forceRefresh || !e.hasBeenFetched() {
|
||||
if err := c.queue.fetchAndStore(ctx, e); err != nil {
|
||||