Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ High-performance Protocol Buffers code generator for Java, optimized for seriali
- **Lazy string/bytes deserialization** — decoded only on access
- **Optimized string handling** — single-copy ASCII fast path via `sun.misc.Unsafe`
- **Protobuf-compatible JSON serialization and deserialization** — `toJson()` / `parseFromJson()` methods
- **Protobuf TextFormat (de)serialization** — opt-in `toTextFormat()` / `parseFromTextFormat()` for compatibility with `com.google.protobuf.TextFormat`
- **No runtime dependencies** — generated code is self-contained
- **Maven and Gradle plugins** for seamless build integration

Expand Down Expand Up @@ -70,11 +71,25 @@ before compilation. Optional configuration:
lightproto {
classPrefix = '' // prefix for generated class names
singleOuterClass = false // wrap all messages in a single outer class
generateTextFormat = false // also generate protobuf TextFormat (de)serialization methods
protocVersion = '4.34.0' // protoc compiler version
// protocPath = '/usr/local/bin/protoc' // use a local protoc binary
}
```

For Maven, the same options are configured under `<configuration>` on the plugin:

```xml
<plugin>
<groupId>io.streamnative.lightproto</groupId>
<artifactId>lightproto-maven-plugin</artifactId>
<configuration>
<generateTextFormat>true</generateTextFormat>
</configuration>
<!-- ... -->
</plugin>
```

### API Example

LightProto generates mutable, reusable objects instead of the Builder pattern used by Google Protobuf:
Expand Down Expand Up @@ -129,6 +144,39 @@ The JSON encoding follows protobuf conventions: lowerCamelCase field names, int6
as strings, enum values as names, and bytes fields as base64. Unknown fields are silently ignored
during parsing, ensuring forward compatibility.

### Protobuf TextFormat (opt-in)

Set `generateTextFormat = true` (Gradle) or `<generateTextFormat>true</generateTextFormat>`
(Maven) to also emit TextFormat (de)serialization on every message. The output is compatible
with `com.google.protobuf.TextFormat.printer()` and `TextFormat.merge()` for backward
compatibility with existing TextFormat data:

```java
// Serialize to a multi-line, indented TextFormat string
String text = md.toTextFormat();
// producer_name: "producer-1"
// sequence_id: 12345
// publish_time: 1711234567890
// property {
// key: "key1"
// value: "value1"
// }

// Or write to a StringBuilder
StringBuilder sb = new StringBuilder();
md.writeTextFormatTo(sb);

// Deserialize from a String, byte[], or ByteBuf
MessageMetadata parsed = new MessageMetadata();
parsed.parseFromTextFormat(text);
```

Differences from JSON: field names are the original proto snake_case, `int64` values are
**not** quoted, enum values are emitted as bare identifiers, and bytes are written as
quoted strings with C-style escapes. The parser tolerates `# … ` comments, single- or
double-quoted strings, angle-bracket sub-messages (`field <…>`), `[v1, v2]` array syntax for
repeated fields, and ignores unknown fields.

### gRPC Integration

LightProto generates `*Grpc.java` service stubs directly from `service` definitions in `.proto`
Expand Down Expand Up @@ -213,6 +261,7 @@ client streaming, and bidirectional streaming.
| Multiple `.proto` files / `import` | ✅ | ✅ |
| `service` / RPC definitions (gRPC stubs) | ✅ | ✅ |
| JSON serialization and deserialization | ✅ | ✅ |
| TextFormat serialization and deserialization (opt-in) | ✅ | ✅ |
| Extensions | ❌ | — |
| `Any`, `Timestamp`, well-known types | ❌ | ❌ |
| `group` (deprecated) | ❌ | — |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ public class LightProto {

public LightProto(ProtoFileDescriptor proto, String protoFileName,
String outerClassName, boolean useOuterClass) {
this(proto, protoFileName, outerClassName, useOuterClass, false);
}

public LightProto(ProtoFileDescriptor proto, String protoFileName,
String outerClassName, boolean useOuterClass,
boolean generateTextFormat) {
this.proto = proto;
this.protoFileName = protoFileName;
this.outerClassName = outerClassName;
Expand All @@ -53,7 +59,7 @@ public LightProto(ProtoFileDescriptor proto, String protoFileName,
" */%n",
LightProtoGenerator.getVersion(), protoFileName);
this.enums = proto.getEnumGroups().stream().map(LightProtoEnum::new).collect(Collectors.toList());
this.messages = proto.getMessages().stream().map(m -> new LightProtoMessage(m, useOuterClass)).collect(Collectors.toList());
this.messages = proto.getMessages().stream().map(m -> new LightProtoMessage(m, useOuterClass, generateTextFormat)).collect(Collectors.toList());
this.services = proto.getServices().stream().map(LightProtoService::new).collect(Collectors.toList());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,19 @@ public void parseJson(PrintWriter w) {
w.format(" %s(_r.readBool());\n", Util.camelCase("set", field.getName()));
}

@Override
public void serializeTextFormat(PrintWriter w) {
w.format("LightProtoCodec.writeTextFormatIndent(_sb, _indent);\n");
w.format("_sb.append(\"%s: \").append(Boolean.toString(%s)).append('\\n');\n",
field.getName(), ccName);
}

@Override
public void parseTextFormat(PrintWriter w) {
w.format(" _r.consumeFieldSeparator();\n");
w.format(" %s(_r.readBool());\n", Util.camelCase("set", field.getName()));
}

@Override
public void equalsCode(PrintWriter w) {
w.format("if (%s != _other.%s) return false;\n", ccName, ccName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,24 @@ public void parseJson(PrintWriter w) {
w.format(" %s(_r.readBase64Bytes());\n", Util.camelCase("set", field.getName()));
}

@Override
public void serializeTextFormat(PrintWriter w) {
w.format("LightProtoCodec.writeTextFormatIndent(_sb, _indent);\n");
w.format("_sb.append(\"%s: \");\n", field.getName());
w.format("if (_%sIdx == -1) {\n", ccName);
w.format(" LightProtoCodec.writeTextFormatBytes(_sb, %s, 0, _%sLen);\n", ccName, ccName);
w.format("} else {\n");
w.format(" LightProtoCodec.writeTextFormatBytes(_sb, _parsedBuffer, _%sIdx, _%sLen);\n", ccName, ccName);
w.format("}\n");
w.format("_sb.append('\\n');\n");
}

@Override
public void parseTextFormat(PrintWriter w) {
w.format(" _r.consumeFieldSeparator();\n");
w.format(" %s(_r.readBytes());\n", Util.camelCase("set", field.getName()));
}

@Override
public void serialize(PrintWriter w) {
w.format("%s;\n", writeTagExpr(tagName()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,19 @@ public void parseJson(PrintWriter w) {
w.format(" if (_v != null) { %s(_v); } }\n", Util.camelCase("set", field.getName()));
}

@Override
public void serializeTextFormat(PrintWriter w) {
w.format("LightProtoCodec.writeTextFormatIndent(_sb, _indent);\n");
w.format("_sb.append(\"%s: \").append(%s.name()).append('\\n');\n", field.getName(), ccName);
}

@Override
public void parseTextFormat(PrintWriter w) {
w.format(" _r.consumeFieldSeparator();\n");
w.format(" { %s _v = %s.valueOf(_r.readIdentifier());\n", field.getJavaType(), field.getJavaType());
w.format(" if (_v != null) { %s(_v); } }\n", Util.camelCase("set", field.getName()));
}

@Override
public void equalsCode(PrintWriter w) {
w.format("if (%s != _other.%s) return false;\n", ccName, ccName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,24 @@ public void fieldClear(PrintWriter w, String enclosingType) {

abstract public void parseJson(PrintWriter w);

/**
* Emit code that writes the complete TextFormat representation of this field.
* Locals available: {@code _sb} (StringBuilder), {@code _indent} (int).
* The emitted code MUST write the indent, field name, value, and trailing newline(s).
*/
public void serializeTextFormat(PrintWriter w) {
throw new UnsupportedOperationException("serializeTextFormat not implemented for " + getClass().getSimpleName());
}

/**
* Emit code that, given a {@code LightProtoCodec.TextFormatReader _r} positioned
* immediately after this field's name, consumes the separator, reads the value,
* and stores it on the message.
*/
public void parseTextFormat(PrintWriter w) {
throw new UnsupportedOperationException("parseTextFormat not implemented for " + getClass().getSimpleName());
}

abstract public void parse(PrintWriter w);

abstract public void copy(PrintWriter w);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ static String getVersion() {
public static List<File> generate(List<ProtoFileDescriptor> descriptors, File outputDirectory,
String classPrefix, boolean useOuterClass,
List<String> fileNames) throws Exception {
return generate(descriptors, outputDirectory, classPrefix, useOuterClass, fileNames, false);
}

public static List<File> generate(List<ProtoFileDescriptor> descriptors, File outputDirectory,
String classPrefix, boolean useOuterClass,
List<String> fileNames,
boolean generateTextFormat) throws Exception {
List<File> generatedFiles = new ArrayList<>();
Set<String> javaPackages = new HashSet<>();

Expand All @@ -67,7 +74,7 @@ public static List<File> generate(List<ProtoFileDescriptor> descriptors, File ou
String javaDir = Joiner.on('/').join(javaPackageName.split("\\."));
Path targetDir = Paths.get(String.format("%s/%s", outputDirectory, javaDir));

LightProto lightProto = new LightProto(proto, fileName, outerClassName, useOuterClass);
LightProto lightProto = new LightProto(proto, fileName, outerClassName, useOuterClass, generateTextFormat);
generatedFiles.addAll(lightProto.generate(targetDir.toFile()));

javaPackages.add(javaPackageName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,174 @@ public void parseJson(PrintWriter w) {
w.format(" }\n");
}

@Override
public void serializeTextFormat(PrintWriter w) {
w.format("for (int _i = 0; _i < _%sCount; _i++) {\n", ccName);
w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent);\n");
w.format(" _sb.append(\"%s {\\n\");\n", field.getName());

// Write key at indent + 1
w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent + 1);\n");
w.format(" _sb.append(\"key: \");\n");
if (isStringKey()) {
w.format(" LightProtoCodec.StringHolder _ksh = _%sKeys[_i];\n", ccName);
w.format(" if (_ksh.s == null) {\n");
w.format(" _ksh.s = LightProtoCodec.readString(_parsedBuffer, _ksh.idx, _ksh.len);\n");
w.format(" }\n");
w.format(" LightProtoCodec.writeTextFormatString(_sb, _ksh.s);\n");
} else if (keyField.getJavaType().equals("boolean")) {
w.format(" _sb.append(Boolean.toString(_%sKeys[_i]));\n", ccName);
} else if (keyField.getJavaType().equals("long")) {
w.format(" _sb.append(Long.toString(_%sKeys[_i]));\n", ccName);
} else {
w.format(" _sb.append(Integer.toString(_%sKeys[_i]));\n", ccName);
}
w.format(" _sb.append('\\n');\n");

// Write value at indent + 1
if (isMessageValue()) {
w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent + 1);\n");
w.format(" _sb.append(\"value {\\n\");\n");
w.format(" _%sValues[_i].writeTextFormatTo(_sb, _indent + 2);\n", ccName);
w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent + 1);\n");
w.format(" _sb.append(\"}\\n\");\n");
} else {
w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent + 1);\n");
w.format(" _sb.append(\"value: \");\n");
if (isStringValue()) {
w.format(" LightProtoCodec.StringHolder _vsh = _%sValues[_i];\n", ccName);
w.format(" if (_vsh.s == null) {\n");
w.format(" _vsh.s = LightProtoCodec.readString(_parsedBuffer, _vsh.idx, _vsh.len);\n");
w.format(" }\n");
w.format(" LightProtoCodec.writeTextFormatString(_sb, _vsh.s);\n");
} else if (isBytesValue()) {
w.format(" LightProtoCodec.BytesHolder _vbh = _%sValues[_i];\n", ccName);
w.format(" if (_vbh.idx == -1) {\n");
w.format(" LightProtoCodec.writeTextFormatBytes(_sb, _vbh.b, 0, _vbh.len);\n");
w.format(" } else {\n");
w.format(" LightProtoCodec.writeTextFormatBytes(_sb, _parsedBuffer, _vbh.idx, _vbh.len);\n");
w.format(" }\n");
} else if (isEnumValue()) {
w.format(" _sb.append(_%sValues[_i].name());\n", ccName);
} else if (valueField.getProtoType().equals("bool")) {
w.format(" _sb.append(Boolean.toString(_%sValues[_i]));\n", ccName);
} else if (valueField.getProtoType().equals("float")) {
w.format(" LightProtoCodec.writeTextFormatFloat(_sb, _%sValues[_i]);\n", ccName);
} else if (valueField.getProtoType().equals("double")) {
w.format(" LightProtoCodec.writeTextFormatDouble(_sb, _%sValues[_i]);\n", ccName);
} else if (valueField.getJavaType().equals("long")) {
w.format(" _sb.append(Long.toString(_%sValues[_i]));\n", ccName);
} else {
w.format(" _sb.append(Integer.toString(_%sValues[_i]));\n", ccName);
}
w.format(" _sb.append('\\n');\n");
}

w.format(" LightProtoCodec.writeTextFormatIndent(_sb, _indent);\n");
w.format(" _sb.append(\"}\\n\");\n");
w.format("}\n");
}

@Override
public void parseTextFormat(PrintWriter w) {
// Each invocation reads ONE entry: '{ key: ... value: ... }'
w.format(" if (_r.tryConsume(':')) {}\n");
w.format(" { char _entryClose = _r.consumeMessageOpen();\n");

// Declare temp vars for key/value
if (isStringKey()) {
w.format(" String _key = \"\";\n");
} else if (keyField.getJavaType().equals("boolean")) {
w.format(" boolean _key = false;\n");
} else if (keyField.getJavaType().equals("long")) {
w.format(" long _key = 0L;\n");
} else {
w.format(" int _key = 0;\n");
}

if (!isMessageValue()) {
if (isStringValue()) {
w.format(" String _val = \"\";\n");
} else if (isBytesValue()) {
w.format(" byte[] _val = new byte[0];\n");
} else if (isEnumValue()) {
w.format(" %s _val = null;\n", valueField.getJavaType());
} else if (valueField.getProtoType().equals("bool")) {
w.format(" boolean _val = false;\n");
} else if (valueField.getProtoType().equals("float")) {
w.format(" float _val = 0.0f;\n");
} else if (valueField.getProtoType().equals("double")) {
w.format(" double _val = 0.0;\n");
} else if (valueField.getJavaType().equals("long")) {
w.format(" long _val = 0L;\n");
} else {
w.format(" int _val = 0;\n");
}
}

if (isMessageValue()) {
// For message values, we can't buffer the inner block — parse directly
// into a freshly-put entry. Maps allow `value` to come before `key`, but
// we resolve that by overwriting the key after.
w.format(" %s _msg = null;\n", valueField.getJavaType());
}

w.format(" while (!_r.atFieldsEnd()) {\n");
w.format(" String _entryFieldName = _r.readIdentifier();\n");
w.format(" if (_entryFieldName.equals(\"key\")) {\n");
w.format(" _r.consumeFieldSeparator();\n");
if (isStringKey()) {
w.format(" _key = _r.readString();\n");
} else if (keyField.getJavaType().equals("boolean")) {
w.format(" _key = _r.readBool();\n");
} else if (keyField.getJavaType().equals("long")) {
w.format(" _key = _r.readLong();\n");
} else {
w.format(" _key = _r.readInt();\n");
}
w.format(" } else if (_entryFieldName.equals(\"value\")) {\n");
if (isMessageValue()) {
w.format(" if (_r.tryConsume(':')) {}\n");
w.format(" _msg = new %s();\n", valueField.getJavaType());
w.format(" _msg.parseFromTextFormat(_r.buf());\n");
} else {
w.format(" _r.consumeFieldSeparator();\n");
if (isStringValue()) {
w.format(" _val = _r.readString();\n");
} else if (isBytesValue()) {
w.format(" _val = _r.readBytes();\n");
} else if (isEnumValue()) {
w.format(" _val = %s.valueOf(_r.readIdentifier());\n", valueField.getJavaType());
} else if (valueField.getProtoType().equals("bool")) {
w.format(" _val = _r.readBool();\n");
} else if (valueField.getProtoType().equals("float")) {
w.format(" _val = _r.readFloat();\n");
} else if (valueField.getProtoType().equals("double")) {
w.format(" _val = _r.readDouble();\n");
} else if (valueField.getJavaType().equals("long")) {
w.format(" _val = _r.readLong();\n");
} else {
w.format(" _val = _r.readInt();\n");
}
}
w.format(" } else {\n");
w.format(" _r.skipValue();\n");
w.format(" }\n");
w.format(" _r.skipOptionalSeparator();\n");
w.format(" }\n");
w.format(" _r.expect(_entryClose);\n");

if (isMessageValue()) {
w.format(" if (_msg != null) { %s(_key).copyFrom(_msg); }\n",
Util.camelCase("put", ccName));
} else if (isEnumValue()) {
w.format(" if (_val != null) { %s(_key, _val); }\n", Util.camelCase("put", ccName));
} else {
w.format(" %s(_key, _val);\n", Util.camelCase("put", ccName));
}
w.format(" }\n");
}

@Override
public void serialize(PrintWriter w) {
w.format("for (int _i = 0; _i < _%sCount; _i++) {\n", ccName);
Expand Down
Loading
Loading