diff --git a/polymarket/high-throughput-paired-basis-maker/config.example.json b/polymarket/high-throughput-paired-basis-maker/config.example.json index d71de3a..795d7a0 100644 --- a/polymarket/high-throughput-paired-basis-maker/config.example.json +++ b/polymarket/high-throughput-paired-basis-maker/config.example.json @@ -21,7 +21,7 @@ "history_fidelity_minutes": 60, "history_fetch_workers": 12, "gamma_markets_url": "https://api.serendb.com/publishers/polymarket-data/markets", - "clob_history_url": "https://api.serendb.com/publishers/polymarket-data/prices-history" + "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" }, "strategy": { "bankroll_usd": 500, diff --git a/polymarket/high-throughput-paired-basis-maker/scripts/agent.py b/polymarket/high-throughput-paired-basis-maker/scripts/agent.py index e3ba4cd..7db9dbb 100644 --- a/polymarket/high-throughput-paired-basis-maker/scripts/agent.py +++ b/polymarket/high-throughput-paired-basis-maker/scripts/agent.py @@ -59,7 +59,7 @@ class BacktestParams: history_interval: str = "max" history_fidelity_minutes: int = 60 gamma_markets_url: str = "https://api.serendb.com/publishers/polymarket-data/markets" - clob_history_url: str = "https://api.serendb.com/publishers/polymarket-data/prices-history" + clob_history_url: str = "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" history_fetch_workers: int = 12 @@ -103,6 +103,13 @@ def _safe_str(value: Any, default: str = "") -> str: return str(value) +def _canonicalize_history_url(url: str) -> str: + trimmed = url.rstrip("/") + if trimmed.endswith("/prices-history"): + return trimmed[: -len("/prices-history")] + "/trades" + return url + + def _safe_bool(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value @@ -175,29 +182,34 @@ def to_backtest_params(config: dict[str, Any]) -> BacktestParams: history_interval=_safe_str(raw.get("history_interval"), "max"), history_fidelity_minutes=max(1, _safe_int(raw.get("history_fidelity_minutes"), 60)), gamma_markets_url=_safe_str(raw.get("gamma_markets_url"), "https://api.serendb.com/publishers/polymarket-data/markets"), - clob_history_url=_safe_str(raw.get("clob_history_url"), "https://api.serendb.com/publishers/polymarket-data/prices-history"), + clob_history_url=_canonicalize_history_url( + _safe_str(raw.get("clob_history_url"), "https://api.serendb.com/publishers/polymarket-trading-serenai/trades") + ), history_fetch_workers=max(1, _safe_int(raw.get("history_fetch_workers"), 12)), ) -def _normalize_history(raw_history: Any, start_ts: int, end_ts: int) -> list[tuple[int, float]]: +def _normalize_history( + raw_history: Any, + start_ts: int, + end_ts: int, + *, + token_id: str = "", +) -> list[tuple[int, float]]: points: list[tuple[int, float]] = [] fallback_points: list[tuple[int, float]] = [] seen: set[int] = set() fallback_seen: set[int] = set() - if not isinstance(raw_history, list): + rows = _extract_history_rows(raw_history) + if not rows: return points - for item in raw_history: - t = -1 - p = -1.0 - if isinstance(item, dict): - t = _safe_int(item.get("t"), -1) - p = _safe_float(item.get("p"), -1.0) - elif isinstance(item, list | tuple) and len(item) >= 2: - t = _safe_int(item[0], -1) - p = _safe_float(item[1], -1.0) + for item in rows: + parsed = _history_point_from_row(item, token_id=token_id) + if parsed is None: + continue + t, p = parsed if t in fallback_seen or not (0.0 <= p <= 1.0): continue fallback_seen.add(t) @@ -227,6 +239,104 @@ def _json_to_list(value: Any) -> list[Any]: return [] +def _extract_history_rows(payload: Any) -> list[Any]: + if isinstance(payload, list): + return payload + if not isinstance(payload, dict): + return [] + for key in ("history", "trades", "data", "items", "results"): + rows = payload.get(key) + if isinstance(rows, list): + return rows + return [] + + +def _coerce_unix_ts(value: Any) -> int: + if isinstance(value, int | float): + ts = int(value) + if ts > 10_000_000_000: + ts //= 1000 + return ts + raw = _safe_str(value, "").strip() + if not raw: + return -1 + if raw.isdigit(): + ts = int(raw) + if ts > 10_000_000_000: + ts //= 1000 + return ts + parsed = _parse_iso_ts(raw) + return parsed if parsed is not None else -1 + + +def _normalize_probability(value: Any) -> float: + p = _safe_float(value, -1.0) + if 1.0 < p <= 100.0: + p /= 100.0 + return p + + +def _row_matches_token(row: dict[str, Any], token_id: str) -> bool: + token = token_id.strip() + if not token: + return True + observed: list[str] = [] + for key in ("token_id", "tokenId", "tokenID", "asset_id", "assetId"): + raw = _safe_str(row.get(key), "").strip() + if raw: + observed.append(raw) + if not observed: + return True + return token in observed + + +def _history_point_from_row(row: Any, token_id: str) -> tuple[int, float] | None: + if isinstance(row, list | tuple) and len(row) >= 2: + ts = _coerce_unix_ts(row[0]) + p = _normalize_probability(row[1]) + if ts < 0 or not (0.0 <= p <= 1.0): + return None + return ts, p + if not isinstance(row, dict): + return None + if not _row_matches_token(row, token_id): + return None + ts = -1 + for key in ( + "t", + "timestamp", + "ts", + "time", + "createdAt", + "created_at", + "updatedAt", + "updated_at", + "matchTime", + ): + ts = _coerce_unix_ts(row.get(key)) + if ts >= 0: + break + if ts < 0: + return None + p = -1.0 + for key in ( + "p", + "price", + "outcomePrice", + "outcome_price", + "probability", + "mid_price", + "midpoint", + ): + candidate = _normalize_probability(row.get(key)) + if 0.0 <= candidate <= 1.0: + p = candidate + break + if p < 0.0: + return None + return ts, p + + def _parse_iso_ts(value: Any) -> int | None: raw = _safe_str(value, "") if not raw: @@ -347,23 +457,22 @@ def _fetch_live_backtest_pairs(p: StrategyParams, bt: BacktestParams, start_ts: candidates_with_cap = candidates[: bt.max_markets] if bt.max_markets > 0 else candidates def _fetch_candidate_history(candidate: dict[str, Any]) -> dict[str, Any] | None: + history_limit = max(bt.min_history_points * 12, 1000) history_query = urlencode( { "market": candidate["token_id"], - "interval": bt.history_interval, - "fidelity": bt.history_fidelity_minutes, + "limit": history_limit, } ) try: payload = _http_get_json(f"{bt.clob_history_url}?{history_query}") except Exception: return None - if not isinstance(payload, dict): - return None history = _normalize_history( - _json_to_list(payload.get("history")), + payload, start_ts=start_ts, end_ts=end_ts, + token_id=candidate["token_id"], ) if len(history) < bt.min_history_points: return None diff --git a/polymarket/liquidity-paired-basis-maker/config.example.json b/polymarket/liquidity-paired-basis-maker/config.example.json index 6c286e6..b19ccc4 100644 --- a/polymarket/liquidity-paired-basis-maker/config.example.json +++ b/polymarket/liquidity-paired-basis-maker/config.example.json @@ -21,7 +21,7 @@ "history_fidelity_minutes": 60, "history_fetch_workers": 4, "gamma_markets_url": "https://api.serendb.com/publishers/polymarket-data/markets", - "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/prices-history" + "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" }, "strategy": { "bankroll_usd": 500, diff --git a/polymarket/liquidity-paired-basis-maker/scripts/agent.py b/polymarket/liquidity-paired-basis-maker/scripts/agent.py index bc97b8d..053777c 100644 --- a/polymarket/liquidity-paired-basis-maker/scripts/agent.py +++ b/polymarket/liquidity-paired-basis-maker/scripts/agent.py @@ -18,7 +18,7 @@ from pathlib import Path from statistics import pstdev from typing import Any -from urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse, urlunparse from urllib.request import Request, urlopen SEREN_POLYMARKET_PUBLISHER_HOST = "api.serendb.com" @@ -78,7 +78,7 @@ class BacktestParams: history_interval: str = "max" history_fidelity_minutes: int = 60 gamma_markets_url: str = f"{SEREN_POLYMARKET_DATA_URL_PREFIX}/markets" - clob_history_url: str = f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/prices-history" + clob_history_url: str = f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/trades" history_fetch_workers: int = 4 @@ -122,6 +122,14 @@ def _safe_str(value: Any, default: str = "") -> str: return str(value) +def _canonicalize_history_url(url: str) -> str: + parsed = urlparse(url) + if parsed.path.endswith("/prices-history"): + path = parsed.path[: -len("/prices-history")] + "/trades" + return urlunparse(parsed._replace(path=path)) + return url + + def _safe_bool(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value @@ -194,29 +202,34 @@ def to_backtest_params(config: dict[str, Any]) -> BacktestParams: history_interval=_safe_str(raw.get("history_interval"), "max"), history_fidelity_minutes=max(1, _safe_int(raw.get("history_fidelity_minutes"), 60)), gamma_markets_url=_safe_str(raw.get("gamma_markets_url"), f"{SEREN_POLYMARKET_DATA_URL_PREFIX}/markets"), - clob_history_url=_safe_str(raw.get("clob_history_url"), f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/prices-history"), + clob_history_url=_canonicalize_history_url( + _safe_str(raw.get("clob_history_url"), f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/trades") + ), history_fetch_workers=max(1, _safe_int(raw.get("history_fetch_workers"), 4)), ) -def _normalize_history(raw_history: Any, start_ts: int, end_ts: int) -> list[tuple[int, float]]: +def _normalize_history( + raw_history: Any, + start_ts: int, + end_ts: int, + *, + token_id: str = "", +) -> list[tuple[int, float]]: points: list[tuple[int, float]] = [] fallback_points: list[tuple[int, float]] = [] seen: set[int] = set() fallback_seen: set[int] = set() - if not isinstance(raw_history, list): + rows = _extract_history_rows(raw_history) + if not rows: return points - for item in raw_history: - t = -1 - p = -1.0 - if isinstance(item, dict): - t = _safe_int(item.get("t"), -1) - p = _safe_float(item.get("p"), -1.0) - elif isinstance(item, list | tuple) and len(item) >= 2: - t = _safe_int(item[0], -1) - p = _safe_float(item[1], -1.0) + for item in rows: + parsed = _history_point_from_row(item, token_id=token_id) + if parsed is None: + continue + t, p = parsed if t in fallback_seen or not (0.0 <= p <= 1.0): continue fallback_seen.add(t) @@ -246,6 +259,119 @@ def _json_to_list(value: Any) -> list[Any]: return [] +def _extract_history_rows(payload: Any) -> list[Any]: + if isinstance(payload, list): + return payload + if not isinstance(payload, dict): + return [] + for key in ("history", "trades", "data", "items", "results"): + rows = payload.get(key) + if isinstance(rows, list): + return rows + body = payload.get("body") + if isinstance(body, list): + return body + if isinstance(body, dict): + for key in ("history", "trades", "data", "items", "results"): + rows = body.get(key) + if isinstance(rows, list): + return rows + return [] + + +def _coerce_unix_ts(value: Any) -> int: + if isinstance(value, int | float): + ts = int(value) + if ts > 10_000_000_000: + ts //= 1000 + return ts + raw = _safe_str(value, "").strip() + if not raw: + return -1 + if raw.isdigit(): + ts = int(raw) + if ts > 10_000_000_000: + ts //= 1000 + return ts + parsed = _parse_iso_ts(raw) + return parsed if parsed is not None else -1 + + +def _normalize_probability(value: Any) -> float: + p = _safe_float(value, -1.0) + if 1.0 < p <= 100.0: + p /= 100.0 + return p + + +def _row_matches_token(row: dict[str, Any], token_id: str) -> bool: + token = token_id.strip() + if not token: + return True + observed: list[str] = [] + for key in ( + "token_id", + "tokenId", + "tokenID", + "asset_id", + "assetId", + "assetID", + ): + raw = _safe_str(row.get(key), "").strip() + if raw: + observed.append(raw) + if not observed: + return True + return token in observed + + +def _history_point_from_row(row: Any, token_id: str) -> tuple[int, float] | None: + if isinstance(row, list | tuple) and len(row) >= 2: + ts = _coerce_unix_ts(row[0]) + p = _normalize_probability(row[1]) + if ts < 0 or not (0.0 <= p <= 1.0): + return None + return ts, p + if not isinstance(row, dict): + return None + if not _row_matches_token(row, token_id): + return None + ts = -1 + for key in ( + "t", + "timestamp", + "ts", + "time", + "createdAt", + "created_at", + "updatedAt", + "updated_at", + "matchTime", + ): + ts = _coerce_unix_ts(row.get(key)) + if ts >= 0: + break + if ts < 0: + return None + p = -1.0 + for key in ( + "p", + "price", + "outcomePrice", + "outcome_price", + "probability", + "mid_price", + "midpoint", + ): + candidate = _normalize_probability(row.get(key)) + if 0.0 <= candidate <= 1.0: + p = candidate + break + if p < 0.0: + return None + return ts, p + + def _parse_iso_ts(value: Any) -> int | None: raw = _safe_str(value, "") if not raw: @@ -590,23 +716,22 @@ def _fetch_live_backtest_pairs(p: StrategyParams, bt: BacktestParams, start_ts: candidates_with_cap = candidates[: bt.max_markets] if bt.max_markets > 0 else candidates def _fetch_candidate_history(candidate: dict[str, Any]) -> dict[str, Any] | None: + history_limit = max(bt.min_history_points * 12, 1000) history_query = urlencode( { "market": candidate["token_id"], - "interval": bt.history_interval, - "fidelity": bt.history_fidelity_minutes, + "limit": history_limit, } ) try: payload = _http_get_json(f"{bt.clob_history_url}?{history_query}") except Exception: return None - if not isinstance(payload, dict): - return None history = _normalize_history( - _json_to_list(payload.get("history")), + payload, start_ts=start_ts, end_ts=end_ts, + token_id=candidate["token_id"], ) if len(history) < bt.min_history_points: return None diff --git a/polymarket/maker-rebate-bot/config.example.json b/polymarket/maker-rebate-bot/config.example.json index 8b9d22a..7da1b07 100644 --- a/polymarket/maker-rebate-bot/config.example.json +++ b/polymarket/maker-rebate-bot/config.example.json @@ -13,7 +13,7 @@ "markets_fetch_limit": 300, "min_history_points": 480, "gamma_markets_url": "https://api.serendb.com/publishers/polymarket-data/markets", - "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/prices-history" + "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" }, "strategy": { "bankroll_usd": 1000, diff --git a/polymarket/maker-rebate-bot/scripts/agent.py b/polymarket/maker-rebate-bot/scripts/agent.py index 11a6c0a..a98de76 100644 --- a/polymarket/maker-rebate-bot/scripts/agent.py +++ b/polymarket/maker-rebate-bot/scripts/agent.py @@ -15,7 +15,7 @@ from pathlib import Path from statistics import pstdev from typing import Any -from urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse, urlunparse from urllib.request import Request, urlopen SEREN_POLYMARKET_PUBLISHER_HOST = "api.serendb.com" @@ -65,7 +65,7 @@ class BacktestParams: markets_fetch_limit: int = 300 min_history_points: int = 480 gamma_markets_url: str = f"{SEREN_POLYMARKET_DATA_URL_PREFIX}/markets" - clob_history_url: str = f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/prices-history" + clob_history_url: str = f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/trades" def parse_args() -> argparse.Namespace: @@ -146,6 +146,14 @@ def _safe_str(value: Any, default: str = "") -> str: return str(value) +def _canonicalize_history_url(url: str) -> str: + parsed = urlparse(url) + if parsed.path.endswith("/prices-history"): + path = parsed.path[: -len("/prices-history")] + "/trades" + return urlunparse(parsed._replace(path=path)) + return url + + def to_params(config: dict[str, Any]) -> StrategyParams: strategy = config.get("strategy", {}) return StrategyParams( @@ -188,9 +196,11 @@ def to_backtest_params(config: dict[str, Any]) -> BacktestParams: backtest.get("gamma_markets_url"), f"{SEREN_POLYMARKET_DATA_URL_PREFIX}/markets", ), - clob_history_url=_safe_str( - backtest.get("clob_history_url"), - f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/prices-history", + clob_history_url=_canonicalize_history_url( + _safe_str( + backtest.get("clob_history_url"), + f"{SEREN_POLYMARKET_TRADING_URL_PREFIX}/trades", + ) ), ) @@ -232,6 +242,24 @@ def _parse_iso_ts(value: Any) -> int | None: return None +def _coerce_unix_ts(value: Any) -> int: + if isinstance(value, int | float): + ts = int(value) + if ts > 10_000_000_000: + ts //= 1000 + return ts + raw = _safe_str(value, "").strip() + if not raw: + return -1 + if raw.isdigit(): + ts = int(raw) + if ts > 10_000_000_000: + ts //= 1000 + return ts + parsed = _parse_iso_ts(raw) + return parsed if parsed is not None else -1 + + def _json_to_list(value: Any) -> list[Any]: if isinstance(value, list): return value @@ -245,6 +273,110 @@ def _json_to_list(value: Any) -> list[Any]: return [] +def _extract_history_rows(payload: Any) -> list[Any]: + if isinstance(payload, list): + return payload + if not isinstance(payload, dict): + return [] + for key in ("history", "trades", "data", "items", "results"): + rows = payload.get(key) + if isinstance(rows, list): + return rows + body = payload.get("body") + if isinstance(body, list): + return body + if isinstance(body, dict): + for key in ("history", "trades", "data", "items", "results"): + rows = body.get(key) + if isinstance(rows, list): + return rows + return [] + + +def _normalize_probability(value: Any) -> float: + p = _safe_float(value, -1.0) + if 1.0 < p <= 100.0: + p /= 100.0 + return p + + +def _row_matches_token(row: dict[str, Any], token_id: str) -> bool: + token = token_id.strip() + if not token: + return True + observed: list[str] = [] + for key in ( + "token_id", + "tokenId", + "tokenID", + "asset_id", + "assetId", + "assetID", + ): + raw = _safe_str(row.get(key), "").strip() + if raw: + observed.append(raw) + asset = row.get("asset") + if isinstance(asset, dict): + for key in ("id", "token_id", "asset_id"): + raw = _safe_str(asset.get(key), "").strip() + if raw: + observed.append(raw) + if not observed: + return True + return token in observed + + +def _history_point_from_row(row: Any, token_id: str) -> tuple[int, float] | None: + if isinstance(row, list | tuple) and len(row) >= 2: + ts = _coerce_unix_ts(row[0]) + p = _normalize_probability(row[1]) + if ts < 0 or not (0.0 <= p <= 1.0): + return None + return ts, p + + if not isinstance(row, dict): + return None + if not _row_matches_token(row, token_id): + return None + + ts = -1 + for key in ( + "t", + "timestamp", + "ts", + "time", + "createdAt", + "created_at", + "updatedAt", + "updated_at", + "matchTime", + ): + ts = _coerce_unix_ts(row.get(key)) + if ts >= 0: + break + if ts < 0: + return None + + p = -1.0 + for key in ( + "p", + "price", + "outcomePrice", + "outcome_price", + "probability", + "mid_price", + "midpoint", + ): + candidate = _normalize_probability(row.get(key)) + if 0.0 <= candidate <= 1.0: + p = candidate + break + if p < 0.0: + return None + return ts, p + + def _is_truthy(value: str | None) -> bool: if value is None: return False @@ -491,32 +623,79 @@ def _http_get_json(url: str, timeout: int = 30) -> dict[str, Any] | list[Any]: def _normalize_history( - history_payload: list[Any], + history_payload: Any, start_ts: int, end_ts: int, + *, + token_id: str = "", + fidelity_minutes: int = 1, ) -> list[tuple[int, float]]: - cleaned: list[tuple[int, float]] = [] + points: list[tuple[int, float]] = [] + fallback_points: list[tuple[int, float]] = [] seen: set[int] = set() - for point in history_payload: - t: int | None = None - p: float | None = None - if isinstance(point, dict): - t = _safe_int(point.get("t"), -1) - p = _safe_float(point.get("p"), -1.0) - elif isinstance(point, list | tuple) and len(point) >= 2: - t = _safe_int(point[0], -1) - p = _safe_float(point[1], -1.0) - - if t is None or p is None: + fallback_seen: set[int] = set() + + for row in _extract_history_rows(history_payload): + parsed = _history_point_from_row(row, token_id=token_id) + if parsed is None: continue - if t < 0 or not (0.0 <= p <= 1.0): + t, p = parsed + if t in fallback_seen: continue + fallback_seen.add(t) + fallback_points.append((t, p)) if t < start_ts or t > end_ts or t in seen: continue seen.add(t) - cleaned.append((t, p)) - cleaned.sort(key=lambda x: x[0]) - return cleaned + points.append((t, p)) + + points.sort(key=lambda pair: pair[0]) + if not points: + fallback_points.sort(key=lambda pair: pair[0]) + points = fallback_points + + if not points: + return [] + if fidelity_minutes <= 1: + return points + + bucket_seconds = max(60, fidelity_minutes * 60) + bucketed: dict[int, tuple[int, float]] = {} + for t, p in points: + bucketed[t // bucket_seconds] = (t, p) + return sorted(bucketed.values(), key=lambda pair: pair[0]) + + +def _fetch_market_history( + backtest_params: BacktestParams, + token_id: str, + start_ts: int, + end_ts: int, +) -> list[tuple[int, float]]: + history_limit = max(backtest_params.min_history_points * 12, 1000) + queries = ( + {"market": token_id, "limit": history_limit}, + {"asset_id": token_id, "limit": history_limit}, + {"token_id": token_id, "limit": history_limit}, + ) + best: list[tuple[int, float]] = [] + for params in queries: + try: + payload = _http_get_json(f"{backtest_params.clob_history_url}?{urlencode(params)}") + except Exception: + continue + history = _normalize_history( + history_payload=payload, + start_ts=start_ts, + end_ts=end_ts, + token_id=token_id, + fidelity_minutes=backtest_params.fidelity_minutes, + ) + if len(history) > len(best): + best = history + if len(best) >= backtest_params.min_history_points: + return best + return best def _load_markets_from_fixture( @@ -537,7 +716,7 @@ def _load_markets_from_fixture( if not isinstance(raw, dict): continue history = _normalize_history( - history_payload=_json_to_list(raw.get("history")), + history_payload=raw.get("history"), start_ts=start_ts, end_ts=end_ts, ) @@ -608,18 +787,9 @@ def _fetch_live_markets( for candidate in candidates: if len(selected) >= strategy_params.markets_max: break - history_query = urlencode( - { - "market": candidate["token_id"], - "interval": "max", - "fidelity": backtest_params.fidelity_minutes, - } - ) - payload = _http_get_json(f"{backtest_params.clob_history_url}?{history_query}") - if not isinstance(payload, dict): - continue - history = _normalize_history( - history_payload=_json_to_list(payload.get("history")), + history = _fetch_market_history( + backtest_params=backtest_params, + token_id=candidate["token_id"], start_ts=start_ts, end_ts=end_ts, ) diff --git a/polymarket/maker-rebate-bot/tests/test_smoke.py b/polymarket/maker-rebate-bot/tests/test_smoke.py index eaaf5c0..e6ab3f0 100644 --- a/polymarket/maker-rebate-bot/tests/test_smoke.py +++ b/polymarket/maker-rebate-bot/tests/test_smoke.py @@ -121,6 +121,7 @@ def test_config_example_uses_seren_polymarket_publisher_urls() -> None: assert backtest.get("clob_history_url", "").startswith( "https://api.serendb.com/publishers/polymarket-trading-serenai/" ) + assert backtest.get("clob_history_url", "").endswith("/trades") def test_backtest_rejects_non_seren_polymarket_data_source(tmp_path: Path) -> None: diff --git a/polymarket/paired-market-basis-maker/config.example.json b/polymarket/paired-market-basis-maker/config.example.json index 26b32bd..a7257fc 100644 --- a/polymarket/paired-market-basis-maker/config.example.json +++ b/polymarket/paired-market-basis-maker/config.example.json @@ -21,7 +21,7 @@ "history_fidelity_minutes": 60, "history_fetch_workers": 12, "gamma_markets_url": "https://api.serendb.com/publishers/polymarket-data/markets", - "clob_history_url": "https://api.serendb.com/publishers/polymarket-data/prices-history" + "clob_history_url": "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" }, "strategy": { "bankroll_usd": 500, diff --git a/polymarket/paired-market-basis-maker/scripts/agent.py b/polymarket/paired-market-basis-maker/scripts/agent.py index a7ee792..e6b1f03 100644 --- a/polymarket/paired-market-basis-maker/scripts/agent.py +++ b/polymarket/paired-market-basis-maker/scripts/agent.py @@ -59,7 +59,7 @@ class BacktestParams: history_interval: str = "max" history_fidelity_minutes: int = 60 gamma_markets_url: str = "https://api.serendb.com/publishers/polymarket-data/markets" - clob_history_url: str = "https://api.serendb.com/publishers/polymarket-data/prices-history" + clob_history_url: str = "https://api.serendb.com/publishers/polymarket-trading-serenai/trades" history_fetch_workers: int = 12 @@ -103,6 +103,13 @@ def _safe_str(value: Any, default: str = "") -> str: return str(value) +def _canonicalize_history_url(url: str) -> str: + trimmed = url.rstrip("/") + if trimmed.endswith("/prices-history"): + return trimmed[: -len("/prices-history")] + "/trades" + return url + + def _safe_bool(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value @@ -175,29 +182,34 @@ def to_backtest_params(config: dict[str, Any]) -> BacktestParams: history_interval=_safe_str(raw.get("history_interval"), "max"), history_fidelity_minutes=max(1, _safe_int(raw.get("history_fidelity_minutes"), 60)), gamma_markets_url=_safe_str(raw.get("gamma_markets_url"), "https://api.serendb.com/publishers/polymarket-data/markets"), - clob_history_url=_safe_str(raw.get("clob_history_url"), "https://api.serendb.com/publishers/polymarket-data/prices-history"), + clob_history_url=_canonicalize_history_url( + _safe_str(raw.get("clob_history_url"), "https://api.serendb.com/publishers/polymarket-trading-serenai/trades") + ), history_fetch_workers=max(1, _safe_int(raw.get("history_fetch_workers"), 12)), ) -def _normalize_history(raw_history: Any, start_ts: int, end_ts: int) -> list[tuple[int, float]]: +def _normalize_history( + raw_history: Any, + start_ts: int, + end_ts: int, + *, + token_id: str = "", +) -> list[tuple[int, float]]: points: list[tuple[int, float]] = [] fallback_points: list[tuple[int, float]] = [] seen: set[int] = set() fallback_seen: set[int] = set() - if not isinstance(raw_history, list): + rows = _extract_history_rows(raw_history) + if not rows: return points - for item in raw_history: - t = -1 - p = -1.0 - if isinstance(item, dict): - t = _safe_int(item.get("t"), -1) - p = _safe_float(item.get("p"), -1.0) - elif isinstance(item, list | tuple) and len(item) >= 2: - t = _safe_int(item[0], -1) - p = _safe_float(item[1], -1.0) + for item in rows: + parsed = _history_point_from_row(item, token_id=token_id) + if parsed is None: + continue + t, p = parsed if t in fallback_seen or not (0.0 <= p <= 1.0): continue fallback_seen.add(t) @@ -227,6 +239,104 @@ def _json_to_list(value: Any) -> list[Any]: return [] +def _extract_history_rows(payload: Any) -> list[Any]: + if isinstance(payload, list): + return payload + if not isinstance(payload, dict): + return [] + for key in ("history", "trades", "data", "items", "results"): + rows = payload.get(key) + if isinstance(rows, list): + return rows + return [] + + +def _coerce_unix_ts(value: Any) -> int: + if isinstance(value, int | float): + ts = int(value) + if ts > 10_000_000_000: + ts //= 1000 + return ts + raw = _safe_str(value, "").strip() + if not raw: + return -1 + if raw.isdigit(): + ts = int(raw) + if ts > 10_000_000_000: + ts //= 1000 + return ts + parsed = _parse_iso_ts(raw) + return parsed if parsed is not None else -1 + + +def _normalize_probability(value: Any) -> float: + p = _safe_float(value, -1.0) + if 1.0 < p <= 100.0: + p /= 100.0 + return p + + +def _row_matches_token(row: dict[str, Any], token_id: str) -> bool: + token = token_id.strip() + if not token: + return True + observed: list[str] = [] + for key in ("token_id", "tokenId", "tokenID", "asset_id", "assetId"): + raw = _safe_str(row.get(key), "").strip() + if raw: + observed.append(raw) + if not observed: + return True + return token in observed + + +def _history_point_from_row(row: Any, token_id: str) -> tuple[int, float] | None: + if isinstance(row, list | tuple) and len(row) >= 2: + ts = _coerce_unix_ts(row[0]) + p = _normalize_probability(row[1]) + if ts < 0 or not (0.0 <= p <= 1.0): + return None + return ts, p + if not isinstance(row, dict): + return None + if not _row_matches_token(row, token_id): + return None + ts = -1 + for key in ( + "t", + "timestamp", + "ts", + "time", + "createdAt", + "created_at", + "updatedAt", + "updated_at", + "matchTime", + ): + ts = _coerce_unix_ts(row.get(key)) + if ts >= 0: + break + if ts < 0: + return None + p = -1.0 + for key in ( + "p", + "price", + "outcomePrice", + "outcome_price", + "probability", + "mid_price", + "midpoint", + ): + candidate = _normalize_probability(row.get(key)) + if 0.0 <= candidate <= 1.0: + p = candidate + break + if p < 0.0: + return None + return ts, p + + def _parse_iso_ts(value: Any) -> int | None: raw = _safe_str(value, "") if not raw: @@ -347,23 +457,22 @@ def _fetch_live_backtest_pairs(p: StrategyParams, bt: BacktestParams, start_ts: candidates_with_cap = candidates[: bt.max_markets] if bt.max_markets > 0 else candidates def _fetch_candidate_history(candidate: dict[str, Any]) -> dict[str, Any] | None: + history_limit = max(bt.min_history_points * 12, 1000) history_query = urlencode( { "market": candidate["token_id"], - "interval": bt.history_interval, - "fidelity": bt.history_fidelity_minutes, + "limit": history_limit, } ) try: payload = _http_get_json(f"{bt.clob_history_url}?{history_query}") except Exception: return None - if not isinstance(payload, dict): - return None history = _normalize_history( - _json_to_list(payload.get("history")), + payload, start_ts=start_ts, end_ts=end_ts, + token_id=candidate["token_id"], ) if len(history) < bt.min_history_points: return None