Skip to content

Commit da16371

Browse files
dfa1claude
dfa1
and
claude
committed
feat: add from-parquet command (hardwood 1.0.0.Beta2)
New hosh-parquet-module with from-parquet (reads Parquet files into records) and to-parquet stub (throws UnsupportedOperationException until hardwood adds write support). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 533291a commit da16371

10 files changed

Lines changed: 456 additions & 0 deletions

File tree

‎main/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@
6161
<version>${project.version}</version>
6262
<scope>runtime</scope>
6363
</dependency>
64+
<dependency>
65+
<groupId>hosh</groupId>
66+
<artifactId>hosh-parquet-module</artifactId>
67+
<version>${project.version}</version>
68+
<scope>runtime</scope>
69+
</dependency>
6470
<dependency>
6571
<groupId>hosh</groupId>
6672
<artifactId>hosh-checksum-module</artifactId>

‎modules/parquet/.jqwik-database‎

4 Bytes
Binary file not shown.

‎modules/parquet/pom.xml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<parent>
4+
<artifactId>hosh-parent</artifactId>
5+
<groupId>hosh</groupId>
6+
<version>0.2.1-SNAPSHOT</version>
7+
<relativePath>../../pom.xml</relativePath>
8+
</parent>
9+
<modelVersion>4.0.0</modelVersion>
10+
<artifactId>hosh-parquet-module</artifactId>
11+
<dependencies>
12+
<dependency>
13+
<groupId>hosh</groupId>
14+
<artifactId>hosh-spi</artifactId>
15+
<version>${project.version}</version>
16+
</dependency>
17+
<dependency>
18+
<groupId>dev.hardwood</groupId>
19+
<artifactId>hardwood-core</artifactId>
20+
</dependency>
21+
<!-- test classpath -->
22+
<dependency>
23+
<groupId>hosh</groupId>
24+
<artifactId>hosh-test-support</artifactId>
25+
<version>${project.version}</version>
26+
<scope>test</scope>
27+
</dependency>
28+
<dependency>
29+
<groupId>hosh</groupId>
30+
<artifactId>hosh-spi-test-support</artifactId>
31+
<version>${project.version}</version>
32+
<scope>test</scope>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.junit.jupiter</groupId>
36+
<artifactId>junit-jupiter</artifactId>
37+
<scope>test</scope>
38+
</dependency>
39+
<dependency>
40+
<groupId>org.assertj</groupId>
41+
<artifactId>assertj-core</artifactId>
42+
<scope>test</scope>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.mockito</groupId>
46+
<artifactId>mockito-core</artifactId>
47+
<scope>test</scope>
48+
</dependency>
49+
<dependency>
50+
<groupId>org.mockito</groupId>
51+
<artifactId>mockito-junit-jupiter</artifactId>
52+
<scope>test</scope>
53+
</dependency>
54+
</dependencies>
55+
</project>

‎modules/parquet/src/main/java/hosh/modules/parquet/ParquetModule.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* MIT License
3+
*
4+
* Copyright (c) 2018-2026 Davide Angelocola
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
package hosh.modules.parquet;
25+
26+
import dev.hardwood.InputFile;
27+
import dev.hardwood.metadata.LogicalType;
28+
import dev.hardwood.reader.ParquetFileReader;
29+
import dev.hardwood.reader.RowReader;
30+
import dev.hardwood.schema.ColumnSchema;
31+
import dev.hardwood.schema.FileSchema;
32+
import hosh.doc.Description;
33+
import hosh.doc.Example;
34+
import hosh.doc.Examples;
35+
import hosh.spi.Command;
36+
import hosh.spi.CommandArguments;
37+
import hosh.spi.CommandName;
38+
import hosh.spi.CommandRegistry;
39+
import hosh.spi.Errors;
40+
import hosh.spi.ExitStatus;
41+
import hosh.spi.InputChannel;
42+
import hosh.spi.Keys;
43+
import hosh.spi.Module;
44+
import hosh.spi.OutputChannel;
45+
import hosh.spi.Records;
46+
import hosh.spi.State;
47+
import hosh.spi.StateAware;
48+
import hosh.spi.Value;
49+
import hosh.spi.Values;
50+
51+
import java.io.IOException;
52+
import java.io.UncheckedIOException;
53+
import java.math.BigDecimal;
54+
import java.nio.charset.StandardCharsets;
55+
import java.nio.file.Files;
56+
import java.nio.file.Path;
57+
import java.time.Instant;
58+
import java.time.LocalDate;
59+
import java.time.LocalTime;
60+
import java.util.List;
61+
import java.util.UUID;
62+
63+
public class ParquetModule implements Module {
64+
65+
@Override
66+
public void initialize(CommandRegistry registry) {
67+
registry.registerCommand(CommandName.constant("from-parquet"), FromParquet::new);
68+
registry.registerCommand(CommandName.constant("to-parquet"), ToParquet::new);
69+
}
70+
71+
@Description("read a Parquet file into records, one record per row")
72+
@Examples({
73+
@Example(description = "read records from a Parquet file", command = "from-parquet data.parquet"),
74+
@Example(description = "read and count records from a Parquet file", command = "from-parquet data.parquet | count"),
75+
})
76+
public static class FromParquet implements Command, StateAware {
77+
78+
private State state;
79+
80+
@Override
81+
public void setState(State state) {
82+
this.state = state;
83+
}
84+
85+
@Override
86+
public ExitStatus run(CommandArguments args, InputChannel in, OutputChannel out, OutputChannel err) {
87+
if (args.size() != 1) {
88+
err.send(Errors.usage("from-parquet file"));
89+
return ExitStatus.error();
90+
}
91+
Path source = args.get(0).asPath(state);
92+
if (!Files.exists(source)) {
93+
err.send(Errors.message("file not found: %s", source));
94+
return ExitStatus.error();
95+
}
96+
if (!Files.isRegularFile(source)) {
97+
err.send(Errors.message("not a regular file: %s", source));
98+
return ExitStatus.error();
99+
}
100+
try (InputFile inputFile = InputFile.of(source);
101+
ParquetFileReader fileReader = ParquetFileReader.open(inputFile)) {
102+
FileSchema schema = fileReader.getFileSchema();
103+
List<ColumnSchema> columns = schema.getColumns();
104+
try (RowReader rowReader = fileReader.rowReader()) {
105+
while (rowReader.hasNext()) {
106+
rowReader.next();
107+
Records.Builder builder = Records.builder();
108+
for (ColumnSchema col : columns) {
109+
int idx = col.columnIndex();
110+
Value value = rowReader.isNull(idx) ? Values.none() : toValue(col, rowReader.getValue(idx));
111+
builder.entry(Keys.of(col.name()), value);
112+
}
113+
out.send(builder.build());
114+
}
115+
}
116+
return ExitStatus.success();
117+
} catch (IOException e) {
118+
throw new UncheckedIOException(e);
119+
}
120+
}
121+
122+
private Value toValue(ColumnSchema col, Object obj) {
123+
return switch (obj) {
124+
case String s -> Values.ofText(s);
125+
case Long l -> Values.ofNumeric(l);
126+
case Integer i -> Values.ofNumeric(i);
127+
case Boolean b -> Values.ofText(b.toString());
128+
case Float f -> Values.ofText(Float.toString(f));
129+
case Double d -> Values.ofText(Double.toString(d));
130+
case LocalDate ld -> Values.ofText(ld.toString());
131+
case LocalTime lt -> Values.ofText(lt.toString());
132+
case Instant ts -> Values.ofInstant(ts);
133+
case BigDecimal bd -> Values.ofText(bd.toPlainString());
134+
case UUID uuid -> Values.ofText(uuid.toString());
135+
case byte[] bytes when col.logicalType() instanceof LogicalType.StringType -> Values.ofText(new String(bytes, StandardCharsets.UTF_8));
136+
case byte[] bytes -> Values.ofBytes(bytes);
137+
default -> Values.ofText(String.valueOf(obj));
138+
};
139+
}
140+
}
141+
142+
@Description("write a stream of records to a Parquet file")
143+
@Examples({
144+
@Example(description = "save ls output to a Parquet file", command = "ls | to-parquet output.parquet"),
145+
})
146+
public static class ToParquet implements Command {
147+
148+
@Override
149+
public ExitStatus run(CommandArguments args, InputChannel in, OutputChannel out, OutputChannel err) {
150+
throw new UnsupportedOperationException("to-parquet: write support not yet available in hardwood");
151+
}
152+
}
153+
}

‎modules/parquet/src/main/java/module-info.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* MIT License
3+
*
4+
* Copyright (c) 2018-2026 Davide Angelocola
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
module hosh.modules.parquet {
25+
requires hosh.spi;
26+
requires dev.hardwood.core;
27+
}

‎modules/parquet/src/main/resources/META-INF/services/hosh.spi.Module‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
hosh.modules.parquet.ParquetModule

‎modules/parquet/src/test/java/hosh/modules/parquet/ParquetModuleFitnessTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* MIT License
3+
*
4+
* Copyright (c) 2018-2026 Davide Angelocola
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
package hosh.modules.parquet;
25+
26+
import com.tngtech.archunit.junit.AnalyzeClasses;
27+
import hosh.test.fitness.UnitTestsFitnessTest;
28+
29+
@AnalyzeClasses(packagesOf = ParquetModule.class)
30+
class ParquetModuleFitnessTest extends UnitTestsFitnessTest {
31+
32+
}

0 commit comments

Comments
 (0)