diff --git a/node/pkg/fetcher/utils.go b/node/pkg/fetcher/utils.go index 4894ead05..3a35a83cc 100644 --- a/node/pkg/fetcher/utils.go +++ b/node/pkg/fetcher/utils.go @@ -13,15 +13,20 @@ import ( "github.com/rs/zerolog/log" ) -func FetchSingle(ctx context.Context, definition *Definition) (float64, error) { - rawResult, err := request.Request[interface{}]( +func FetchSingle(ctx context.Context, definition *Definition, reqOpts ...request.RequestOption) (float64, error) { + requestOptions := []request.RequestOption{ request.WithEndpoint(*definition.Url), request.WithHeaders(definition.Headers), - request.WithTimeout(10*time.Second), - ) + request.WithTimeout(10 * time.Second), + } + + requestOptions = append(requestOptions, reqOpts...) + + rawResult, err := request.Request[interface{}](requestOptions...) if err != nil { return 0, err } + return reducer.Reduce(rawResult, definition.Reducers) } diff --git a/node/pkg/por/app.go b/node/pkg/por/app.go index f7fa611b8..f9e2f4914 100644 --- a/node/pkg/por/app.go +++ b/node/pkg/por/app.go @@ -12,14 +12,17 @@ import ( "sync" "time" + "github.com/rs/zerolog/log" + "bisonai.com/miko/node/pkg/chain/helper" chainUtils "bisonai.com/miko/node/pkg/chain/utils" + "bisonai.com/miko/node/pkg/common/types" + "bisonai.com/miko/node/pkg/db" errorSentinel "bisonai.com/miko/node/pkg/error" "bisonai.com/miko/node/pkg/fetcher" "bisonai.com/miko/node/pkg/secrets" "bisonai.com/miko/node/pkg/utils/request" "bisonai.com/miko/node/pkg/utils/retrier" - "github.com/rs/zerolog/log" ) const ( @@ -34,10 +37,47 @@ var urls = map[string]urlEntry{ "peg-por": { "/{CHAIN}/peg-{CHAIN}.por.json", "/{CHAIN}/peg.por.json", + false, }, "gp": { "/{CHAIN}/gp-{CHAIN}.json", "/{CHAIN}/gp.json", + false, + }, + "aapl": { + "/{CHAIN}/aapl-{CHAIN}.json", + "/{CHAIN}/aapl.json", + true, + }, + "amzn": { + "/{CHAIN}/amzn-{CHAIN}.json", + "/{CHAIN}/amzn.json", + true, + }, + "googl": { + "/{CHAIN}/googl-{CHAIN}.json", + "/{CHAIN}/googl.json", + true, + }, + "meta": { + "/{CHAIN}/meta-{CHAIN}.json", + "/{CHAIN}/meta.json", + true, + }, + "msft": { + "/{CHAIN}/msft-{CHAIN}.json", + "/{CHAIN}/msft.json", + true, + }, + "nvda": { + "/{CHAIN}/nvda-{CHAIN}.json", + "/{CHAIN}/nvda.json", + true, + }, + "tsla": { + "/{CHAIN}/tsla-{CHAIN}.json", + "/{CHAIN}/tsla.json", + true, }, } @@ -84,6 +124,7 @@ func New(ctx context.Context) (*app, error) { definition: d, adapter: ad, aggregator: ag, + useProxy: u.useProxy, } entries[n] = e @@ -104,9 +145,15 @@ func New(ctx context.Context) (*app, error) { return nil, err } + proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil) + if err != nil { + return nil, err + } + return &app{ entries: entries, kaiaHelper: chainHelper, + proxies: proxies, }, nil } @@ -183,7 +230,7 @@ func (a *app) startJob(ctx context.Context, entry entry) { } func (a *app) execute(ctx context.Context, e entry) error { - v, err := fetcher.FetchSingle(ctx, e.definition) + v, err := fetcher.FetchSingle(ctx, e.definition, a.buildRequestOpts(e)...) if err != nil { return err } @@ -319,3 +366,26 @@ func (a *app) getRoundId(ctx context.Context, e entry) (uint32, error) { return RoundID, nil } + +func (a *app) getNextProxy() *string { + a.Lock() + defer a.Unlock() + + if len(a.proxies) == 0 { + return nil + } + + proxy := a.proxies[a.proxyIdx%len(a.proxies)].GetProxyUrl() + a.proxyIdx++ + return &proxy +} + +func (a *app) buildRequestOpts(e entry) []request.RequestOption { + opts := []request.RequestOption{} + if e.useProxy { + if p := a.getNextProxy(); p != nil && *p != "" { + opts = append(opts, request.WithProxy(*p)) + } + } + return opts +} diff --git a/node/pkg/por/types.go b/node/pkg/por/types.go index 41067c1aa..5d3c0b7de 100644 --- a/node/pkg/por/types.go +++ b/node/pkg/por/types.go @@ -3,9 +3,11 @@ package por import ( "encoding/json" "math/big" + "sync" "time" "bisonai.com/miko/node/pkg/chain/helper" + "bisonai.com/miko/node/pkg/common/types" "bisonai.com/miko/node/pkg/fetcher" ) @@ -37,12 +39,17 @@ const ( type app struct { entries map[string]entry kaiaHelper *helper.ChainHelper + + proxies []types.Proxy + proxyIdx int + sync.Mutex } type entry struct { definition *fetcher.Definition adapter adaptor aggregator aggregator + useProxy bool } type feed struct { @@ -73,4 +80,5 @@ type lastInfo struct { type urlEntry struct { adapterEndpoint, aggregatorEndpoint string + useProxy bool }