Do partial processing if action is ProcessPartial and request body size exceeds limit

In other words, if the request body size is equal or less than the RequestBodySizeLimit,
the request processor requires a well-formed XML and does not allow a partial JSON,
even if SecRequestBodyLimitAction is set to ProcessPartial.

Author: hnakamur

headers/modsecurity/transaction.h

@@ -645,6 +645,11 @@ class Transaction : public TransactionAnchoredVariables, public TransactionSecMa
      * the web server (connector) log.
      */
     void *m_logCbData;
+
+    /**
+     * Whether the request body was bigger than RequestBodyLimit.
+     */
+    bool m_requestBodyLimitExceeded;
 };
 
 

src/request_body_processor/json.cc

@@ -29,7 +29,7 @@ namespace RequestBodyProcessor {
 static const double json_depth_limit_default = 10000.0;
 static const char* json_depth_limit_exceeded_msg = ". Parsing depth limit exceeded";
 
-JSON::JSON(Transaction *transaction, unsigned int allow_partial_values)
+JSON::JSON(Transaction *transaction)
     : m_transaction(transaction),
     m_handle(NULL),
     m_current_key(""),
@@ -69,8 +69,6 @@ JSON::JSON(Transaction *transaction, unsigned int allow_partial_values)
      * TODO: make UTF8 validation optional, as it depends on Content-Encoding
      */
     m_handle = yajl_alloc(&callbacks, NULL, this);
-
-    yajl_config(m_handle, yajl_allow_partial_values, allow_partial_values);
 }
 
 
@@ -84,7 +82,8 @@ JSON::~JSON() {
 }
 
 
-bool JSON::init() {
+bool JSON::init(unsigned int allow_partial_values) {
+    yajl_config(m_handle, yajl_allow_partial_values, allow_partial_values);
     return true;
 }
 

src/request_body_processor/json.h

@@ -57,10 +57,10 @@ class JSONContainerMap : public JSONContainer {
 
 class JSON {
  public:
-    explicit JSON(Transaction *transaction, unsigned int allow_partial_values = 0);
+    explicit JSON(Transaction *transaction);
     ~JSON();
 
-    static bool init();
+    bool init(unsigned int allow_partial_values = 0);
     bool processChunk(const char *buf, unsigned int size, std::string *err);
     bool complete(std::string *err);
 

src/request_body_processor/xml.cc

@@ -149,8 +149,8 @@ extern "C" {
     }
 }
 
-XML::XML(Transaction *transaction, bool require_well_formed)
-    : m_transaction(transaction), m_require_well_formed(require_well_formed) {
+XML::XML(Transaction *transaction)
+    : m_transaction(transaction), m_require_well_formed(false) {
     m_data.doc = NULL;
     m_data.parsing_ctx = NULL;
     m_data.sax_handler = NULL;
@@ -171,7 +171,8 @@ XML::~XML() {
     }
 }
 
-bool XML::init() {
+bool XML::init(bool require_well_formed) {
+    m_require_well_formed = require_well_formed;
     //xmlParserInputBufferCreateFilenameFunc entity;
     if (m_transaction->m_rules->m_secXMLExternalEntity
         == RulesSetProperties::TrueConfigBoolean) {

src/request_body_processor/xml.h

@@ -85,9 +85,9 @@ typedef struct xml_data xml_data;
 
 class XML {
  public:
-    explicit XML(Transaction *transaction, bool require_well_formed = true);
+    explicit XML(Transaction *transaction);
     ~XML();
-    bool init();
+    bool init(bool require_well_formed = true);
     bool processChunk(const char *buf, unsigned int size, std::string *err);
     bool complete(std::string *err);
     static xmlParserInputBufferPtr unloadExternalEntity(const char *URI,

src/transaction.cc

@@ -138,20 +138,19 @@ Transaction::Transaction(ModSecurity *ms, RulesSet *rules, const char *id,
         ms->m_session_collection, ms->m_user_collection,
         ms->m_resource_collection),
 #ifdef WITH_LIBXML2
-    m_xml(std::make_unique<RequestBodyProcessor::XML>(this,
-        this->m_rules->m_requestBodyLimitAction != RulesSet::BodyLimitAction::ProcessPartialBodyLimitAction)),
+    m_xml(std::make_unique<RequestBodyProcessor::XML>(this)),
 #else
     m_xml(nullptr),
 #endif
 #ifdef WITH_YAJL
-    m_json(std::make_unique<RequestBodyProcessor::JSON>(this,
-        this->m_rules->m_requestBodyLimitAction == RulesSet::BodyLimitAction::ProcessPartialBodyLimitAction)),
+    m_json(std::make_unique<RequestBodyProcessor::JSON>(this)),
 #else
     m_json(nullptr),
 #endif
     m_secRuleEngine(RulesSetProperties::PropertyNotSetRuleEngine),
     m_secXMLParseXmlIntoArgs(rules->m_secXMLParseXmlIntoArgs),
     m_logCbData(logCbData),
+    m_requestBodyLimitExceeded(false),
     TransactionAnchoredVariables(this) {
     m_variableUrlEncodedError.set("0", 0);
     m_variableMscPcreError.set("0", 0);
@@ -689,27 +688,33 @@ int Transaction::processRequestBody() {
     std::unique_ptr<std::string> a = m_variableRequestHeaders.resolveFirst(
         "Content-Type");
 
+    bool is_process_partial = (m_rules->m_requestBodyLimitAction
+        == RulesSet::BodyLimitAction::ProcessPartialBodyLimitAction);
+
     bool requestBodyNoFilesLimitExceeded = false;
     if ((m_requestBodyType == WWWFormUrlEncoded) ||
         (m_requestBodyProcessor == JSONRequestBody) ||
         (m_requestBodyProcessor == XMLRequestBody)) {
         if ((m_rules->m_requestBodyNoFilesLimit.m_set)
             && (m_requestBody.str().size() > m_rules->m_requestBodyNoFilesLimit.m_value)) {
-            m_variableReqbodyError.set("1", 0);
-            m_variableReqbodyErrorMsg.set("Request body excluding files is bigger than the maximum expected.", 0);
-            m_variableInboundDataError.set("1", m_variableOffset);
-            ms_dbg(5, "Request body excluding files is bigger than the maximum expected. Limit: " \
-                + std::to_string(m_rules->m_requestBodyNoFilesLimit.m_value));
+            if (!is_process_partial) {
+                m_variableReqbodyError.set("1", 0);
+                m_variableReqbodyErrorMsg.set("Request body excluding files is bigger than the maximum expected.", 0);
+                m_variableInboundDataError.set("1", m_variableOffset);
+                ms_dbg(5, "Request body excluding files is bigger than the maximum expected. Limit: " \
+                    + std::to_string(m_rules->m_requestBodyNoFilesLimit.m_value));
+            }
             requestBodyNoFilesLimitExceeded = true;
-	}
+	    }
     }
 
 #ifdef WITH_LIBXML2
     if (m_requestBodyProcessor == XMLRequestBody) {
         // large size might cause issues in the parsing itself; omit if exceeded
-        if (!requestBodyNoFilesLimitExceeded) {
+        if (!requestBodyNoFilesLimitExceeded || is_process_partial) {
             std::string error;
-            if (m_xml->init() == true) {
+            bool require_well_formed = !(is_process_partial && m_requestBodyLimitExceeded);
+            if (m_xml->init(require_well_formed) == true) {
                 m_xml->processChunk(m_requestBody.str().c_str(),
                     m_requestBody.str().size(),
                     &error);
@@ -735,12 +740,13 @@ int Transaction::processRequestBody() {
     if (m_requestBodyProcessor == JSONRequestBody) {
 #endif
         // large size might cause issues in the parsing itself; omit if exceeded
-        if (!requestBodyNoFilesLimitExceeded) {
+        if (!requestBodyNoFilesLimitExceeded || is_process_partial) {
             std::string error;
             if (m_rules->m_requestBodyJsonDepthLimit.m_set) {
                 m_json->setMaxDepth(m_rules->m_requestBodyJsonDepthLimit.m_value);
             }
-            if (m_json->init() == true) {
+            unsigned int allow_partial_values = is_process_partial && m_requestBodyLimitExceeded;
+            if (m_json->init(allow_partial_values) == true) {
                 m_json->processChunk(m_requestBody.str().c_str(),
                     m_requestBody.str().size(),
                     &error);
@@ -930,6 +936,7 @@ int Transaction::appendRequestBody(const unsigned char *buf, size_t len) {
 
     if (this->m_rules->m_requestBodyLimit.m_value > 0
         && this->m_rules->m_requestBodyLimit.m_value < len + current_size) {
+        m_requestBodyLimitExceeded = true;
         m_variableInboundDataError.set("1", m_variableOffset);
         ms_dbg(5, "Request body is bigger than the maximum expected.");
 

test/regression/regression.cc

@@ -311,6 +311,7 @@ void perform_unit_test(const ModSecurityTest<RegressionTest> &test,
         modsec_transaction.appendResponseBody(
             (unsigned char *)t->response_body.c_str(),
             t->response_body.size());
+
         modsec_transaction.processResponseBody();
         actions(&r, &modsec_transaction, &context.m_server_log);