Skip to content

Commit 915e2a7

Browse files
mrdrivingduckclaude
mrdrivingduck
and
claude
committed
feat(scan): add IN/NOT IN operator support to predicate pushdown
Extend predicate pushdown to handle COMPARE_IN and COMPARE_NOT_IN expressions, enabling queries with IN (...) and NOT IN (...) clauses to benefit from predicate pushdown optimization. Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent d36b804 commit 915e2a7

2 files changed

Lines changed: 118 additions & 0 deletions

File tree

‎src/paimon_storage/paimon_scan.cpp‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include "duckdb/planner/expression/bound_comparison_expression.hpp"
3030
#include "duckdb/planner/expression/bound_conjunction_expression.hpp"
3131
#include "duckdb/planner/expression/bound_constant_expression.hpp"
32+
#include "duckdb/planner/expression/bound_operator_expression.hpp"
3233
#include "duckdb/planner/operator/logical_get.hpp"
3334

3435
#include "paimon_catalog.hpp"
@@ -104,6 +105,72 @@ static std::shared_ptr<paimon::Predicate> TryConvertComparison(const BoundCompar
104105
}
105106
}
106107

108+
static std::shared_ptr<paimon::Predicate> TryConvertOperator(const BoundOperatorExpression &op, LogicalGet &get) {
109+
// Validate children count per operator type.
110+
switch (op.type) {
111+
case ExpressionType::COMPARE_IN:
112+
case ExpressionType::COMPARE_NOT_IN:
113+
D_ASSERT(op.children.size() >= 2);
114+
break;
115+
default:
116+
return nullptr;
117+
}
118+
119+
// We can only deal with column ref as the first child.
120+
if (op.children[0]->GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) {
121+
return nullptr;
122+
}
123+
124+
// Get column index and name.
125+
auto filter_binding_idx = op.children[0]->Cast<BoundColumnRefExpression>().binding.column_index;
126+
auto col_idx = get.GetColumnIds()[filter_binding_idx];
127+
auto paimon_type = PaimonTypeUtils::ConvertFieldType(get.GetColumnType(col_idx));
128+
auto field_index = col_idx.GetPrimaryIndex();
129+
auto &field_name = get.GetColumnName(col_idx);
130+
131+
switch (op.type) {
132+
case ExpressionType::COMPARE_IN:
133+
case ExpressionType::COMPARE_NOT_IN: {
134+
// Collect literals from children[1..n].
135+
std::vector<paimon::Literal> literals;
136+
for (idx_t i = 1; i < op.children.size(); i++) {
137+
if (op.children[i]->GetExpressionClass() != ExpressionClass::BOUND_CONSTANT) {
138+
// Best effort pushdown.
139+
if (op.type == ExpressionType::COMPARE_NOT_IN) {
140+
continue;
141+
} else {
142+
return nullptr;
143+
}
144+
}
145+
146+
auto val = op.children[i]->Cast<BoundConstantExpression>().value;
147+
auto literal = PaimonTypeUtils::ConvertLiteral(val, paimon_type);
148+
if (!literal) {
149+
// Same reason as above: best effort pushdown.
150+
if (op.type == ExpressionType::COMPARE_NOT_IN) {
151+
continue;
152+
} else {
153+
return nullptr;
154+
}
155+
}
156+
literals.push_back(std::move(literal.value()));
157+
}
158+
159+
if (literals.empty()) {
160+
return nullptr;
161+
}
162+
163+
if (op.type == ExpressionType::COMPARE_IN) {
164+
return paimon::PredicateBuilder::In(field_index, field_name, paimon_type, literals);
165+
} else {
166+
return paimon::PredicateBuilder::NotIn(field_index, field_name, paimon_type, literals);
167+
}
168+
}
169+
default:
170+
return nullptr;
171+
}
172+
}
173+
107174
// Forward declaration for mutual recursion with TryConvertConjunction.
108175
static std::shared_ptr<paimon::Predicate> TryConvertExpression(const Expression &expr, LogicalGet &get);
109176

@@ -149,6 +216,8 @@ static std::shared_ptr<paimon::Predicate> TryConvertExpression(const Expression
149216
return TryConvertComparison(expr.Cast<BoundComparisonExpression>(), get);
150217
case ExpressionClass::BOUND_CONJUNCTION:
151218
return TryConvertConjunction(expr.Cast<BoundConjunctionExpression>(), get);
219+
case ExpressionClass::BOUND_OPERATOR:
220+
return TryConvertOperator(expr.Cast<BoundOperatorExpression>(), get);
152221
default:
153222
return nullptr;
154223
}

‎test/sql/predicate_pushdown.test‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,52 @@ Iris 3 2
8181
query III
8282
SELECT f0, f1, f2 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE (f1 = 1 AND f2 = 3) OR (f1 = 3 AND f2 = 3);
8383
----
84+
85+
# predicate pushdown: IN
86+
query II
87+
SELECT f0, f1 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f1 IN (1, 3);
88+
----
89+
Alice 1
90+
Bob 1
91+
Cathy 1
92+
Grace 3
93+
Henry 3
94+
Iris 3
95+
96+
# predicate pushdown: IN with single value
97+
query II
98+
SELECT f0, f2 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f2 IN (0);
99+
----
100+
Alice 0
101+
David 0
102+
Grace 0
103+
104+
# predicate pushdown: IN with no match
105+
query II
106+
SELECT f0, f1 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f1 IN (4, 5);
107+
----
108+
109+
# predicate pushdown: NOT IN
110+
query II
111+
SELECT f0, f1 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f1 NOT IN (1, 3);
112+
----
113+
David 2
114+
Eve 2
115+
Frank 2
116+
117+
# predicate pushdown: IN combined with AND
118+
query III
119+
SELECT f0, f1, f2 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f1 IN (1, 2) AND f2 = 0;
120+
----
121+
Alice 1 0
122+
David 2 0
123+
124+
# predicate pushdown: IN combined with OR
125+
query III
126+
SELECT f0, f1, f2 FROM paimon_scan('./data/testdb.db/testtbl', manifest_format='orc') WHERE f1 IN (1) OR f2 IN (2);
127+
----
128+
Alice 1 0
129+
Bob 1 1
130+
Cathy 1 2
131+
Frank 2 2
132+
Iris 3 2

0 commit comments

Comments
 (0)