Skip to content

Commit b953a17

Browse files
daniel-kmiecikdaniel-kmiecik
committed
fix: parse large file with resolve option set to true
1 parent 5625506 commit b953a17

File tree

5 files changed

+86222
-90
lines changed

5 files changed

+86222
-90
lines changed

‎modules/swagger-parser-v3/src/main/java/io/swagger/v3/parser/reference/ReferenceVisitor.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import io.swagger.v3.parser.util.RemoteUrl;
1919
import org.apache.commons.lang3.StringUtils;
2020
import org.slf4j.LoggerFactory;
21+
import org.yaml.snakeyaml.LoaderOptions;
22+
import org.yaml.snakeyaml.Yaml;
2123

2224
import java.util.HashMap;
2325
import java.util.HashSet;
@@ -35,6 +37,8 @@ public class ReferenceVisitor extends AbstractVisitor {
3537
protected Reference reference;
3638
protected DereferencerContext context;
3739
private PermittedUrlsChecker permittedUrlsChecker;
40+
private final LoaderOptions loaderOptions;
41+
3842

3943
public ReferenceVisitor(
4044
Reference reference,
@@ -46,6 +50,7 @@ public ReferenceVisitor(
4650
this.visited = visited;
4751
this.visitedMap = visitedMap;
4852
this.context = null;
53+
this.loaderOptions = new LoaderOptions();
4954
}
5055

5156
public ReferenceVisitor(
@@ -61,6 +66,7 @@ public ReferenceVisitor(
6166
this.context = context;
6267
this.permittedUrlsChecker = new PermittedUrlsChecker(context.getParseOptions().getRemoteRefAllowList(),
6368
context.getParseOptions().getRemoteRefBlockList());
69+
this.loaderOptions = new LoaderOptions();
6470
}
6571

6672
public String toBaseURI(String uri) throws Exception{
@@ -301,7 +307,25 @@ public JsonNode findAnchor(JsonNode root, String anchor) {
301307

302308
public JsonNode deserializeIntoTree(String content) throws Exception {
303309
boolean isJson = content.trim().startsWith("{");
304-
return isJson ? Json31.mapper().readTree(content) : Yaml31.mapper().readTree(content);
310+
if (isJson) {
311+
return Json31.mapper().readTree(content);
312+
}
313+
Yaml yaml = getYaml();
314+
315+
Object yamlObject = yaml.load(content);
316+
return Yaml31.mapper().valueToTree(yamlObject);
317+
}
318+
319+
private Yaml getYaml() {
320+
Yaml yaml;
321+
String yamlCodePoints = System.getProperty("maxYamlCodePoints");
322+
if (yamlCodePoints != null && !yamlCodePoints.isEmpty() && StringUtils.isNumeric(yamlCodePoints)) {
323+
loaderOptions.setCodePointLimit(Integer.parseInt(yamlCodePoints));
324+
yaml = new Yaml(loaderOptions);
325+
} else {
326+
yaml = new Yaml();
327+
}
328+
return yaml;
305329
}
306330

307331
public JsonNode parse(String absoluteUri, List<AuthorizationValue> auths) throws Exception {

‎modules/swagger-parser-v3/src/main/java/io/swagger/v3/parser/util/DeserializationUtils.java‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ public static class Options {
4141
private Long maxYamlReferences = System.getProperty("maxYamlReferences") == null ? 10000000L : Long.parseLong(System.getProperty("maxYamlReferences"));
4242
private boolean validateYamlInput = System.getProperty("validateYamlInput") == null || Boolean.parseBoolean(System.getProperty("validateYamlInput"));
4343
private boolean yamlCycleCheck = System.getProperty("yamlCycleCheck") == null || Boolean.parseBoolean(System.getProperty("yamlCycleCheck"));
44-
private Integer maxYamlCodePoints = System.getProperty("maxYamlCodePoints") == null ? 3 * 1024 * 1024 : Integer.parseInt(System.getProperty("maxYamlCodePoints"));
4544
private Integer maxYamlAliasesForCollections = System.getProperty("maxYamlAliasesForCollections") == null ? Integer.MAX_VALUE : Integer.parseInt(System.getProperty("maxYamlAliasesForCollections"));
4645
private boolean yamlAllowRecursiveKeys = System.getProperty("yamlAllowRecursiveKeys") == null || Boolean.parseBoolean(System.getProperty("yamlAllowRecursiveKeys"));
4746

@@ -79,11 +78,7 @@ public void setYamlCycleCheck(boolean yamlCycleCheck) {
7978
}
8079

8180
public Integer getMaxYamlCodePoints() {
82-
return maxYamlCodePoints;
83-
}
84-
85-
public void setMaxYamlCodePoints(Integer maxYamlCodePoints) {
86-
this.maxYamlCodePoints = maxYamlCodePoints;
81+
return System.getProperty("maxYamlCodePoints") == null ? 3 * 1024 * 1024 : Integer.parseInt(System.getProperty("maxYamlCodePoints"));
8782
}
8883

8984
public Integer getMaxYamlAliasesForCollections() {
@@ -103,7 +98,7 @@ public void setYamlAllowRecursiveKeys(boolean yamlAllowRecursiveKeys) {
10398
}
10499
}
105100

106-
private static Options options = new Options();
101+
private static final Options options = new Options();
107102

108103
private static final Logger LOGGER = LoggerFactory.getLogger(DeserializationUtils.class);
109104

@@ -145,7 +140,6 @@ protected void addImplicitResolvers() {
145140
addImplicitResolver(Tag.MERGE, MERGE, "<");
146141
addImplicitResolver(Tag.NULL, NULL, "~nN\0");
147142
addImplicitResolver(Tag.NULL, EMPTY, null);
148-
// addImplicitResolver(Tag.TIMESTAMP, TIMESTAMP, "0123456789");
149143
}
150144
}
151145

@@ -249,15 +243,15 @@ public static JsonNode readYamlTree(String contents, ParseOptions parseOptions,
249243
return Json.mapper().convertValue(yaml.load(contents), JsonNode.class);
250244
}
251245
try {
252-
org.yaml.snakeyaml.Yaml yaml = null;
246+
org.yaml.snakeyaml.Yaml yaml;
253247
if (options.isValidateYamlInput()) {
254248
yaml = buildSnakeYaml(new CustomSnakeYamlConstructor());
255249
} else {
256250
yaml = buildSnakeYaml(new SafeConstructor(buildLoaderOptions()));
257251
}
258252
Object o = yaml.load(contents);
259253
if (options.isValidateYamlInput()) {
260-
boolean res = exceedsLimits(o, null, new Integer(0), new IdentityHashMap<Object, Long>(), deserializationUtilsResult);
254+
boolean res = exceedsLimits(o, null, 0, new IdentityHashMap<>(), deserializationUtilsResult);
261255
if (res) {
262256
LOGGER.warn("Error converting snake-parsed yaml to JsonNode");
263257
return getYaml30Mapper().readTree(contents);
@@ -270,8 +264,8 @@ public static JsonNode readYamlTree(String contents, ParseOptions parseOptions,
270264
//
271265
}
272266
return Json.mapper().convertValue(o, JsonNode.class);
273-
} catch (Throwable e) {
274-
LOGGER.warn("Error snake-parsing yaml content", e);
267+
} catch (Exception e) {
268+
LOGGER.warn(e.getMessage(), e);
275269
if (deserializationUtilsResult != null) {
276270
deserializationUtilsResult.message(e.getMessage());
277271
}
@@ -302,8 +296,7 @@ public static org.yaml.snakeyaml.Yaml buildSnakeYaml(BaseConstructor constructor
302296
}
303297
try {
304298
LoaderOptions loaderOptions = buildLoaderOptions();
305-
org.yaml.snakeyaml.Yaml yaml = new org.yaml.snakeyaml.Yaml(constructor, new Representer(new DumperOptions()), new DumperOptions(), loaderOptions, new CustomResolver());
306-
return yaml;
299+
return new org.yaml.snakeyaml.Yaml(constructor, new Representer(new DumperOptions()), new DumperOptions(), loaderOptions, new CustomResolver());
307300
} catch (Exception e) {
308301
//
309302
LOGGER.error("error building snakeYaml", e);
@@ -479,8 +472,8 @@ public Object getSingleData(Class<?> type) {
479472
throw new SnakeException("StackOverflow safe-checking yaml content (maxDepth " + options.getMaxYamlDepth() + ")", e);
480473
} catch (DuplicateKeyException e) {
481474
throw new SnakeException(e.getProblem().replace("found duplicate key", "Duplicate field"));
482-
} catch (Throwable e) {
483-
throw new SnakeException("Exception safe-checking yaml content (maxDepth " + options.getMaxYamlDepth() + ", maxYamlAliasesForCollections " + options.getMaxYamlAliasesForCollections() + ")", e);
475+
} catch (Exception e) {
476+
throw new SnakeException(e.getMessage() + "; Max code points: " + options.getMaxYamlCodePoints(), e);
484477
}
485478
}
486479
}

‎modules/swagger-parser-v3/src/test/java/io/swagger/v3/parser/reference/ReferenceVisitorTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package io.swagger.v3.parser.reference;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import org.testng.annotations.Test;
5+
import org.yaml.snakeyaml.error.YAMLException;
6+
7+
import java.io.IOException;
8+
import java.io.InputStream;
9+
import java.nio.charset.StandardCharsets;
10+
11+
import static org.junit.Assert.assertFalse;
12+
import static org.testng.Assert.assertNotNull;
13+
14+
public class ReferenceVisitorTest {
15+
16+
@Test
17+
public void largeFileShouldBeParsedByJacksonLibraryWhenCodeLimitIsSet() throws Exception {
18+
System.setProperty("maxYamlCodePoints", "10000000");
19+
ReferenceVisitor visitor = new ReferenceVisitor(null, null, null, null);
20+
String resourceName = "/issue2059/largeFile.yaml";
21+
String content = readResourceAsString(resourceName);
22+
23+
JsonNode jsonNode = visitor.deserializeIntoTree(content);
24+
25+
assertNotNull(content);
26+
assertFalse(content.isEmpty());
27+
assertNotNull(jsonNode);
28+
System.clearProperty("maxYamlCodePoints");
29+
}
30+
31+
@Test
32+
public void largeFileShouldNotBeParsedByJacksonLibraryWhenCodeLimitIsNotSet() throws Exception {
33+
ReferenceVisitor visitor = new ReferenceVisitor(null, null, null, null);
34+
String resourceName = "/issue2059/largeFile.yaml";
35+
String content = readResourceAsString(resourceName);
36+
37+
try {
38+
visitor.deserializeIntoTree(content);
39+
// Fail if no exception is thrown
40+
org.testng.Assert.fail("Expected YAMLException was not thrown");
41+
} catch (YAMLException ex) {
42+
org.testng.Assert.assertTrue(
43+
ex.getMessage() != null &&
44+
ex.getMessage().contains("The incoming YAML document exceeds the limit: 3145728 code points."),
45+
"Unexpected YAMLException message: " + ex.getMessage()
46+
);
47+
}
48+
}
49+
50+
private String readResourceAsString(String resourceName) throws IOException {
51+
try (InputStream is = this.getClass().getResourceAsStream(resourceName)) {
52+
if (is == null) {
53+
throw new IOException("Resource not found: " + resourceName);
54+
}
55+
java.io.ByteArrayOutputStream buffer = new java.io.ByteArrayOutputStream();
56+
byte[] data = new byte[4096];
57+
int nRead;
58+
while ((nRead = is.read(data, 0, data.length)) != -1) {
59+
buffer.write(data, 0, nRead);
60+
}
61+
return new String(buffer.toByteArray(), StandardCharsets.UTF_8);
62+
}
63+
}
64+
}

0 commit comments

Comments
 (0)