Skip to content

[por] minor updates for m7 integration #2466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
2 commits merged into from
Sep 15, 2025
Merged

[por] minor updates for m7 integration #2466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
da14b20
feat: updates for m7
rikiisworking Sep 12, 2025
a556a6b
chore: include into url defs
rikiisworking Sep 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions node/pkg/fetcher/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,20 @@ import (
"github.com/rs/zerolog/log"
)

func FetchSingle(ctx context.Context, definition *Definition) (float64, error) {
rawResult, err := request.Request[interface{}](
func FetchSingle(ctx context.Context, definition *Definition, reqOpts ...request.RequestOption) (float64, error) {
requestOptions := []request.RequestOption{
request.WithEndpoint(*definition.Url),
request.WithHeaders(definition.Headers),
Comment on lines 18 to 19
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai bot Sep 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Guard against nil URL to prevent panic.
*definition.Url will panic if Url is nil (e.g., malformed config). Confirm upstream guarantees or add a defensive check returning an error.

Run to spot any definitions missing url:

🏁 Script executed:

#!/bin/bash
rg -nP -C2 '"url"\s*:\s*(null|""|\{\s*\})' -g '!**/node_modules/**' -S

Length of output: 148

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf "PWD: %s\n\n" "$(pwd)"
printf "Top-level files:\n"
ls -la | sed -n '1,120p' || true

printf "\nCheck exact path: node/pkg/fetcher/utils.go\n"
if [ -f node/pkg/fetcher/utils.go ]; then
  printf "FOUND: node/pkg/fetcher/utils.go\n\n"
  nl -ba node/pkg/fetcher/utils.go | sed -n '1,200p'
else
  printf "Not found at node/pkg/fetcher/utils.go\n\n"
  printf "Listing all utils.go files (maxdepth 6):\n"
  find . -maxdepth 6 -type f -name 'utils.go' -print || true

  printf "\nSearching for relevant patterns across the repo (grep fallback; excluding node_modules and .git):\n"
  grep -RIn --exclude-dir=node_modules --exclude-dir=.git -E 'definition\.Url|definition\.url|\*\s*definition\.Url|WithEndpoint\(|WithHeaders\(' . || true

  printf "\nSearching for Go struct fields named 'Url' (Url string or Url *string) in .go files:\n"
  grep -RIn --exclude-dir=node_modules --exclude-dir=.git --include='*.go' -E '\bUrl\b\s+(\*?string)' . || true

  printf "\nSearching for types named *Definition (type .*Definition) to locate 'definition' struct declarations:\n"
  grep -RIn --exclude-dir=node_modules --exclude-dir=.git --include='*.go' -E 'type\s+[A-Za-z0-9_]*Definition\b' . || true
fi

Length of output: 1528

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "PWD: $(pwd)"
echo

if [ -f node/pkg/fetcher/utils.go ]; then
  echo "----- node/pkg/fetcher/utils.go (first 200 lines) -----"
  awk '{printf("%4d: %s\n", NR, $0)}' node/pkg/fetcher/utils.go | sed -n '1,200p'
else
  echo "node/pkg/fetcher/utils.go not found"
fi

echo
echo "=== Repo searches (rg fallback to grep) ==="

rg_cmd() {
  if command -v rg >/dev/null 2>&1; then
    rg -n --hidden --no-ignore -S "$@" --glob '!**/node_modules/**' || true
  else
    grep -RIn --exclude-dir=node_modules --exclude-dir=.git -E "$@" . || true
  fi
}

echo
echo "-- occurrences of 'definition.Url' --"
rg_cmd 'definition\.Url'

echo
echo "-- occurrences of '*definition.Url' (explicit deref) --"
rg_cmd '\*definition\.Url'

echo
echo "-- usages of WithEndpoint( --"
rg_cmd 'WithEndpoint\('

echo
echo "-- Go declarations that include 'Url' (to find field type) --"
rg_cmd '\bUrl\b'

echo
echo "-- type names ending with Definition --"
rg_cmd 'type\s+[A-Za-z0-9_]*Definition\b'

Length of output: 3793

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "PWD: $(pwd)"
echo

echo "1) List Go files under node/pkg (top 200):"
find node/pkg -type f -name '*.go' -not -path '*/node_modules/*' -not -path '*/.git/*' -print | sed -n '1,200p' || true
echo

echo "2) Find 'type ...Definition struct' declarations:"
find . -type f -name '*.go' -not -path '*/node_modules/*' -not -path '*/.git/*' -print0 | xargs -0 grep -nE --line-number 'type\s+[A-Za-z0-9_]*Definition\s+struct' || true
echo

echo "3) Find any 'Definition' symbol occurrences (to locate the type):"
find . -type f -name '*.go' -not -path '*/node_modules/*' -not -path '*/.git/*' -print0 | xargs -0 grep -nE --line-number '\bDefinition\b' || true
echo

echo "4) Search for 'Url' field declarations and occurrences in Go files:"
find . -type f -name '*.go' -not -path '*/node_modules/*' -not -path '*/.git/*' -print0 | xargs -0 grep -nE --line-number '\bUrl\b' || true
echo

echo "5) Show files in node/pkg/utils/request (if present) and search for WithEndpoint signature:"
if [ -d node/pkg/utils/request ]; then
  echo "Files in node/pkg/utils/request:"
  find node/pkg/utils/request -maxdepth 2 -type f -name '*.go' -print || true
  echo
  echo "Searching for 'WithEndpoint' definition/usage in repo:"
  find . -type f -name '*.go' -not -path '*/node_modules/*' -not -path '*/.git/*' -print0 | xargs -0 grep -nE --line-number 'WithEndpoint' || true
  echo
  echo "If found, print the request package files (first 200 lines each):"
  for f in $(find node/pkg/utils/request -type f -name '*.go' -print); do
    echo "---- $f ----"
    sed -n '1,200p' "$f" || true
    echo
  done
else
  echo "node/pkg/utils/request not found"
fi

Length of output: 31334

Guard against nil definition.Url (avoid panic).

Definition.Url is a *string (node/pkg/fetcher/types.go:97) but is dereferenced without a nil-check. Fix by validating/enforcing url on input or adding defensive checks that return an error instead of panicking. Relevant call sites to update: node/pkg/fetcher/utils.go:18, node/pkg/fetcher/fetcher.go:170,174, node/script/test_single_fetch/main.go:28.

🤖 Prompt for AI Agents 
In node/pkg/fetcher/utils.go around lines 18-19 the code dereferences
definition.Url (a *string) without checking for nil which can cause a panic; add
a defensive nil-check and return an error if Url is nil (or validate earlier and
propagate a clear error) rather than dereferencing. Update the other call sites
mentioned (node/pkg/fetcher/fetcher.go lines ~170/174 and
node/script/test_single_fetch/main.go:28) to handle the new error/validation
(either validate definition.Url before calling or handle the returned error) so
no code ever blindly dereferences the pointer.

request.WithTimeout(10*time.Second),
)
request.WithTimeout(10 * time.Second),
}

requestOptions = append(requestOptions, reqOpts...)

rawResult, err := request.Request[interface{}](requestOptions...)
if err != nil {
return 0, err
}

return reducer.Reduce(rawResult, definition.Reducers)
}

Expand Down
74 changes: 72 additions & 2 deletions node/pkg/por/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,17 @@ import (
"sync"
"time"

"github.com/rs/zerolog/log"

"bisonai.com/miko/node/pkg/chain/helper"
chainUtils "bisonai.com/miko/node/pkg/chain/utils"
"bisonai.com/miko/node/pkg/common/types"
"bisonai.com/miko/node/pkg/db"
errorSentinel "bisonai.com/miko/node/pkg/error"
"bisonai.com/miko/node/pkg/fetcher"
"bisonai.com/miko/node/pkg/secrets"
"bisonai.com/miko/node/pkg/utils/request"
"bisonai.com/miko/node/pkg/utils/retrier"
"github.com/rs/zerolog/log"
)

const (
Expand All @@ -34,10 +37,47 @@ var urls = map[string]urlEntry{
"peg-por": {
"/{CHAIN}/peg-{CHAIN}.por.json",
"/{CHAIN}/peg.por.json",
false,
},
"gp": {
"/{CHAIN}/gp-{CHAIN}.json",
"/{CHAIN}/gp.json",
false,
},
"aapl": {
"/{CHAIN}/aapl-{CHAIN}.json",
"/{CHAIN}/aapl.json",
true,
},
"amzn": {
"/{CHAIN}/amzn-{CHAIN}.json",
"/{CHAIN}/amzn.json",
true,
},
"googl": {
"/{CHAIN}/googl-{CHAIN}.json",
"/{CHAIN}/googl.json",
true,
},
"meta": {
"/{CHAIN}/meta-{CHAIN}.json",
"/{CHAIN}/meta.json",
true,
},
"msft": {
"/{CHAIN}/msft-{CHAIN}.json",
"/{CHAIN}/msft.json",
true,
},
"nvda": {
"/{CHAIN}/nvda-{CHAIN}.json",
"/{CHAIN}/nvda.json",
true,
},
"tsla": {
"/{CHAIN}/tsla-{CHAIN}.json",
"/{CHAIN}/tsla.json",
true,
},
}

Expand Down Expand Up @@ -84,6 +124,7 @@ func New(ctx context.Context) (*app, error) {
definition: d,
adapter: ad,
aggregator: ag,
useProxy: u.useProxy,
}

entries[n] = e
Expand All @@ -104,9 +145,15 @@ func New(ctx context.Context) (*app, error) {
return nil, err
}

proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil)
if err != nil {
return nil, err
}
Comment on lines +148 to +151
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai bot Sep 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Don’t hard-fail on proxy DB errors; degrade gracefully.
If the proxies table is missing or DB is transiently down, POR can still run without proxies. Recommend warning + proceed with no proxies.

- proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil)
- if err != nil {
-   return nil, err
- }
+ proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil)
+ if err != nil {
+   log.Warn().Err(err).Msg("failed to load proxies; continuing without proxies")
+   proxies = nil
+ }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil)
if err != nil {
return nil, err
}
proxies, err := db.QueryRows[types.Proxy](ctx, "SELECT * FROM proxies", nil)
if err != nil {
log.Warn().Err(err).Msg("failed to load proxies; continuing without proxies")
proxies = nil
}
🤖 Prompt for AI Agents 
In node/pkg/por/app.go around lines 148 to 151, the current code returns on any
DB error when querying proxies; change this to degrade gracefully by catching
the error, logging a warning that includes the error details and context (e.g.,
"unable to load proxies, proceeding without proxies"), and then set proxies to
an empty slice (or nil) so the rest of POR can continue; only return an error
for truly fatal conditions, not for missing/temporary proxy DB failures.


return &app{
entries: entries,
kaiaHelper: chainHelper,
proxies: proxies,
}, nil
}

Expand Down Expand Up @@ -183,7 +230,7 @@ func (a *app) startJob(ctx context.Context, entry entry) {
}

func (a *app) execute(ctx context.Context, e entry) error {
v, err := fetcher.FetchSingle(ctx, e.definition)
v, err := fetcher.FetchSingle(ctx, e.definition, a.buildRequestOpts(e)...)
if err != nil {
return err
}
Expand Down Expand Up @@ -319,3 +366,26 @@ func (a *app) getRoundId(ctx context.Context, e entry) (uint32, error) {

return RoundID, nil
}

func (a *app) getNextProxy() *string {
a.Lock()
defer a.Unlock()

if len(a.proxies) == 0 {
return nil
}

proxy := a.proxies[a.proxyIdx%len(a.proxies)].GetProxyUrl()
a.proxyIdx++
return &proxy
}

func (a *app) buildRequestOpts(e entry) []request.RequestOption {
opts := []request.RequestOption{}
if e.useProxy {
if p := a.getNextProxy(); p != nil && *p != "" {
opts = append(opts, request.WithProxy(*p))
}
}
return opts
}
8 changes: 8 additions & 0 deletions node/pkg/por/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package por
import (
"encoding/json"
"math/big"
"sync"
"time"

"bisonai.com/miko/node/pkg/chain/helper"
"bisonai.com/miko/node/pkg/common/types"
"bisonai.com/miko/node/pkg/fetcher"
)

Expand Down Expand Up @@ -37,12 +39,17 @@ const (
type app struct {
entries map[string]entry
kaiaHelper *helper.ChainHelper

proxies []types.Proxy
proxyIdx int
sync.Mutex
}

type entry struct {
definition *fetcher.Definition
adapter adaptor
aggregator aggregator
useProxy bool
}

type feed struct {
Expand Down Expand Up @@ -73,4 +80,5 @@ type lastInfo struct {

type urlEntry struct {
adapterEndpoint, aggregatorEndpoint string
useProxy bool
}