Skip to content

Commit d9d5193

Browse files
andinuxclaude
andcommitted
fix(network): parse check responses with a dynamic token budget
The JSON extract helpers (json_extract_object_raw/string/int/bool/ array_size) parsed into a fixed jsmntok_t[64]. A batched cursor-spool /check response wraps several chunks in {"data":{"chunks":[...]}} and easily exceeds 64 tokens (≈7 tokens per signed-URL chunk), so jsmn_parse returned JSMN_ERROR_NOMEM and json_extract_object_raw("data") returned NULL. The drain then looked for "chunks" at the top level — where it is nested under the un-unwrapped "data" — found nothing, took the "up to date" path, reset the spool cursor to 0, and re-requested the same batch forever (single-sync drain never completed). Route all five extract helpers through json_parse_tokens_alloc, which sizes the token array to the document. This removes the 64-token cliff for the "data" unwrap and for the final/nextCursor reads on a large data object, with no fixed cap on response size. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent d09b206 commit d9d5193

1 file changed

Lines changed: 66 additions & 71 deletions

File tree

src/network/network.c

Lines changed: 66 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -807,8 +807,6 @@ char *network_authentication_token (const char *key, const char *value) {
807807

808808
// MARK: - JSON helpers (jsmn) -
809809

810-
#define JSMN_MAX_TOKENS 64
811-
812810
static bool jsmn_token_eq(const char *json, const jsmntok_t *tok, const char *s) {
813811
return (tok->type == JSMN_STRING &&
814812
(int)strlen(s) == tok->end - tok->start &&
@@ -912,77 +910,75 @@ static char *json_unescape_string(const char *src, int len) {
912910
static char *json_extract_string(const char *json, size_t json_len, const char *key) {
913911
if (!json || json_len == 0 || !key) return NULL;
914912

915-
jsmn_parser parser;
916-
jsmntok_t tokens[JSMN_MAX_TOKENS];
917-
jsmn_init(&parser);
918-
int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS);
919-
if (ntokens < 1) return NULL;
913+
int ntokens = 0;
914+
jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens);
915+
if (!tokens) return NULL;
920916

917+
char *result = NULL;
921918
int i = jsmn_find_key(json, tokens, ntokens, key);
922-
if (i < 0 || i + 1 >= ntokens) return NULL;
923-
924-
jsmntok_t *val = &tokens[i + 1];
925-
if (val->type != JSMN_STRING) return NULL;
926-
927-
return json_unescape_string(json + val->start, val->end - val->start);
919+
if (i >= 0 && i + 1 < ntokens) {
920+
jsmntok_t *val = &tokens[i + 1];
921+
if (val->type == JSMN_STRING)
922+
result = json_unescape_string(json + val->start, val->end - val->start);
923+
}
924+
cloudsync_memory_free(tokens);
925+
return result;
928926
}
929927

930928
static int64_t json_extract_int(const char *json, size_t json_len, const char *key, int64_t default_value) {
931929
if (!json || json_len == 0 || !key) return default_value;
932930

933-
jsmn_parser parser;
934-
jsmntok_t tokens[JSMN_MAX_TOKENS];
935-
jsmn_init(&parser);
936-
int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS);
937-
if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return default_value;
931+
int ntokens = 0;
932+
jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens);
933+
if (!tokens) return default_value;
938934

939-
int i = jsmn_find_key(json, tokens, ntokens, key);
940-
if (i < 0 || i + 1 >= ntokens) return default_value;
941-
942-
jsmntok_t *val = &tokens[i + 1];
943-
if (val->type != JSMN_PRIMITIVE) return default_value;
944-
945-
return strtoll(json + val->start, NULL, 10);
935+
int64_t result = default_value;
936+
if (tokens[0].type == JSMN_OBJECT) {
937+
int i = jsmn_find_key(json, tokens, ntokens, key);
938+
if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_PRIMITIVE)
939+
result = strtoll(json + tokens[i + 1].start, NULL, 10);
940+
}
941+
cloudsync_memory_free(tokens);
942+
return result;
946943
}
947944

948945
static bool json_extract_bool(const char *json, size_t json_len, const char *key, bool default_value) {
949946
if (!json || json_len == 0 || !key) return default_value;
950947

951-
jsmn_parser parser;
952-
jsmntok_t tokens[JSMN_MAX_TOKENS];
953-
jsmn_init(&parser);
954-
int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS);
955-
if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return default_value;
956-
957-
int i = jsmn_find_key(json, tokens, ntokens, key);
958-
if (i < 0 || i + 1 >= ntokens) return default_value;
959-
960-
jsmntok_t *val = &tokens[i + 1];
961-
if (val->type != JSMN_PRIMITIVE) return default_value;
962-
963-
// JSON booleans (true/false) and numeric flags (1/0) are both accepted.
964-
char c = json[val->start];
965-
if (c == 't' || c == 'T') return true;
966-
if (c == 'f' || c == 'F' || c == 'n' || c == 'N') return false;
967-
return strtoll(json + val->start, NULL, 10) != 0;
948+
int ntokens = 0;
949+
jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens);
950+
if (!tokens) return default_value;
951+
952+
bool result = default_value;
953+
if (tokens[0].type == JSMN_OBJECT) {
954+
int i = jsmn_find_key(json, tokens, ntokens, key);
955+
if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_PRIMITIVE) {
956+
// JSON booleans (true/false) and numeric flags (1/0) are both accepted.
957+
char c = json[tokens[i + 1].start];
958+
if (c == 't' || c == 'T') result = true;
959+
else if (c == 'f' || c == 'F' || c == 'n' || c == 'N') result = false;
960+
else result = strtoll(json + tokens[i + 1].start, NULL, 10) != 0;
961+
}
962+
}
963+
cloudsync_memory_free(tokens);
964+
return result;
968965
}
969966

970967
static int json_extract_array_size(const char *json, size_t json_len, const char *key) {
971968
if (!json || json_len == 0 || !key) return -1;
972969

973-
jsmn_parser parser;
974-
jsmntok_t tokens[JSMN_MAX_TOKENS];
975-
jsmn_init(&parser);
976-
int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS);
977-
if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return -1;
978-
979-
int i = jsmn_find_key(json, tokens, ntokens, key);
980-
if (i < 0 || i + 1 >= ntokens) return -1;
981-
982-
jsmntok_t *val = &tokens[i + 1];
983-
if (val->type != JSMN_ARRAY) return -1;
970+
int ntokens = 0;
971+
jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens);
972+
if (!tokens) return -1;
984973

985-
return val->size;
974+
int result = -1;
975+
if (tokens[0].type == JSMN_OBJECT) {
976+
int i = jsmn_find_key(json, tokens, ntokens, key);
977+
if (i >= 0 && i + 1 < ntokens && tokens[i + 1].type == JSMN_ARRAY)
978+
result = tokens[i + 1].size;
979+
}
980+
cloudsync_memory_free(tokens);
981+
return result;
986982
}
987983

988984
// Escape a string for safe embedding as a JSON string value (without surrounding quotes).
@@ -1025,25 +1021,24 @@ static char *json_escape_string(const char *src) {
10251021
static char *json_extract_object_raw(const char *json, size_t json_len, const char *key) {
10261022
if (!json || json_len == 0 || !key) return NULL;
10271023

1028-
jsmn_parser parser;
1029-
jsmntok_t tokens[JSMN_MAX_TOKENS];
1030-
jsmn_init(&parser);
1031-
int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS);
1032-
if (ntokens < 1) return NULL;
1024+
int ntokens = 0;
1025+
jsmntok_t *tokens = json_parse_tokens_alloc(json, json_len, &ntokens);
1026+
if (!tokens) return NULL;
10331027

1028+
char *out = NULL;
10341029
int i = jsmn_find_key(json, tokens, ntokens, key);
1035-
if (i < 0 || i + 1 >= ntokens) return NULL;
1036-
1037-
jsmntok_t *val = &tokens[i + 1];
1038-
if (val->type != JSMN_OBJECT) return NULL;
1039-
1040-
int len = val->end - val->start;
1041-
if (len <= 0) return NULL;
1042-
1043-
char *out = cloudsync_memory_zeroalloc(len + 1);
1044-
if (!out) return NULL;
1045-
memcpy(out, json + val->start, len);
1046-
out[len] = '\0';
1030+
if (i >= 0 && i + 1 < ntokens) {
1031+
jsmntok_t *val = &tokens[i + 1];
1032+
int len = val->end - val->start;
1033+
if (val->type == JSMN_OBJECT && len > 0) {
1034+
out = cloudsync_memory_zeroalloc(len + 1);
1035+
if (out) {
1036+
memcpy(out, json + val->start, len);
1037+
out[len] = '\0';
1038+
}
1039+
}
1040+
}
1041+
cloudsync_memory_free(tokens);
10471042
return out;
10481043
}
10491044

0 commit comments

Comments
 (0)