Herb: Implement Syntax Tree Diff Engine

.gitignore

@@ -128,6 +128,9 @@ src/include/ast/ast_pretty_print.h
 src/include/errors.h
 src/include/lib/hb_foreach.h
 src/parser/match_tags.c
+src/diff/herb_diff_helpers.c
+src/diff/herb_diff_nodes.c
+src/diff/herb_hash_tree.c
 src/visitor.c
 wasm/error_helpers.cpp
 wasm/error_helpers.h

ext/herb/extconf.rb

@@ -47,6 +47,7 @@
 $VPATH << "$(srcdir)/../../src/analyze"
 $VPATH << "$(srcdir)/../../src/analyze/action_view"
 $VPATH << "$(srcdir)/../../src/ast"
+$VPATH << "$(srcdir)/../../src/diff"
 $VPATH << "$(srcdir)/../../src/lexer"
 $VPATH << "$(srcdir)/../../src/location"
 $VPATH << "$(srcdir)/../../src/parser"

ext/herb/extension.c

@@ -404,6 +404,89 @@ static VALUE Herb_version(VALUE self) {
 #endif
 }
 
+typedef struct {
+  AST_DOCUMENT_NODE_T* old_root;
+  AST_DOCUMENT_NODE_T* new_root;
+  herb_diff_result_T* diff_result;
+  hb_allocator_T old_allocator;
+  hb_allocator_T new_allocator;
+  hb_allocator_T diff_allocator;
+} diff_args_T;
+
+static VALUE rb_create_diff_operation(const herb_diff_operation_T* operation) {
+  VALUE cDiffOperation = rb_const_get(mHerb, rb_intern("DiffOperation"));
+
+  VALUE type = ID2SYM(rb_intern(herb_diff_operation_type_to_string(operation->type)));
+
+  VALUE path_array = rb_ary_new_capa(operation->path.depth);
+  for (uint16_t index = 0; index < operation->path.depth; index++) {
+    rb_ary_push(path_array, UINT2NUM(operation->path.indices[index]));
+  }
+
+  VALUE old_node = operation->old_node != NULL ? rb_node_from_c_struct((AST_NODE_T*) operation->old_node) : Qnil;
+  VALUE new_node = operation->new_node != NULL ? rb_node_from_c_struct((AST_NODE_T*) operation->new_node) : Qnil;
+
+  VALUE args[] = {
+    type, path_array, old_node, new_node, UINT2NUM(operation->old_index), UINT2NUM(operation->new_index)
+  };
+
+  return rb_class_new_instance(6, args, cDiffOperation);
+}
+
+static VALUE diff_convert_body(VALUE arg) {
+  diff_args_T* args = (diff_args_T*) arg;
+  herb_diff_result_T* diff_result = args->diff_result;
+
+  VALUE cDiffResult = rb_const_get(mHerb, rb_intern("DiffResult"));
+
+  size_t operation_count = herb_diff_operation_count(diff_result);
+  VALUE operations_array = rb_ary_new_capa((long) operation_count);
+
+  for (size_t index = 0; index < operation_count; index++) {
+    const herb_diff_operation_T* operation = herb_diff_operation_at(diff_result, index);
+    rb_ary_push(operations_array, rb_create_diff_operation(operation));
+  }
+
+  VALUE result_args[] = { diff_result->trees_identical ? Qtrue : Qfalse, operations_array };
+
+  return rb_class_new_instance(2, result_args, cDiffResult);
+}
+
+static VALUE diff_cleanup(VALUE arg) {
+  diff_args_T* args = (diff_args_T*) arg;
+
+  if (args->old_root != NULL) { ast_node_free((AST_NODE_T*) args->old_root, &args->old_allocator); }
+  if (args->new_root != NULL) { ast_node_free((AST_NODE_T*) args->new_root, &args->new_allocator); }
+
+  hb_allocator_destroy(&args->diff_allocator);
+  hb_allocator_destroy(&args->old_allocator);
+  hb_allocator_destroy(&args->new_allocator);
+
+  return Qnil;
+}
+
+static VALUE Herb_diff(int argc, VALUE* argv, VALUE self) {
+  VALUE old_source, new_source;
+  rb_scan_args(argc, argv, "2", &old_source, &new_source);
+
+  char* old_string = (char*) check_string(old_source);
+  char* new_string = (char*) check_string(new_source);
+
+  diff_args_T args = { 0 };
+
+  parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+
+  if (!hb_allocator_init(&args.old_allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+  if (!hb_allocator_init(&args.new_allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+  if (!hb_allocator_init(&args.diff_allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+
+  args.old_root = herb_parse(old_string, &parser_options, &args.old_allocator);
+  args.new_root = herb_parse(new_string, &parser_options, &args.new_allocator);
+  args.diff_result = herb_diff(args.old_root, args.new_root, &args.diff_allocator);
+
+  return rb_ensure(diff_convert_body, (VALUE) &args, diff_cleanup, (VALUE) &args);
+}
+
 __attribute__((__visibility__("default"))) void Init_herb(void) {
   mHerb = rb_define_module("Herb");
   cPosition = rb_define_class_under(mHerb, "Position", rb_cObject);
@@ -425,4 +508,5 @@ __attribute__((__visibility__("default"))) void Init_herb(void) {
   rb_define_singleton_method(mHerb, "arena_stats", Herb_arena_stats, -1);
   rb_define_singleton_method(mHerb, "leak_check", Herb_leak_check, 1);
   rb_define_singleton_method(mHerb, "version", Herb_version, 0);
+  rb_define_singleton_method(mHerb, "diff", Herb_diff, -1);
 }

java/herb_jni.c

@@ -1,8 +1,10 @@
 #include "herb_jni.h"
 #include "extension_helpers.h"
+#include "nodes.h"
 
 #include "../../src/include/extract.h"
 #include "../../src/include/herb.h"
+#include "../../src/include/diff/herb_diff.h"
 #include "../../src/include/lib/hb_allocator.h"
 #include "../../src/include/lib/hb_buffer.h"
 
@@ -246,6 +248,83 @@ Java_org_herb_Herb_parseRuby(JNIEnv* env, jclass clazz, jstring source) {
   return result;
 }
 
+JNIEXPORT jobject JNICALL
+Java_org_herb_Herb_diff(JNIEnv* env, jclass clazz, jstring old_source, jstring new_source) {
+  const char* old_src = (*env)->GetStringUTFChars(env, old_source, 0);
+  const char* new_src = (*env)->GetStringUTFChars(env, new_source, 0);
+
+  hb_allocator_T old_allocator;
+  hb_allocator_T new_allocator;
+  hb_allocator_T diff_allocator;
+
+  if (!hb_allocator_init(&old_allocator, HB_ALLOCATOR_ARENA)
+      || !hb_allocator_init(&new_allocator, HB_ALLOCATOR_ARENA)
+      || !hb_allocator_init(&diff_allocator, HB_ALLOCATOR_ARENA)) {
+    (*env)->ReleaseStringUTFChars(env, old_source, old_src);
+    (*env)->ReleaseStringUTFChars(env, new_source, new_src);
+    return NULL;
+  }
+
+  parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+
+  AST_DOCUMENT_NODE_T* old_root = herb_parse(old_src, &parser_options, &old_allocator);
+  AST_DOCUMENT_NODE_T* new_root = herb_parse(new_src, &parser_options, &new_allocator);
+  herb_diff_result_T* diff_result = herb_diff(old_root, new_root, &diff_allocator);
+
+  jclass diff_result_class = (*env)->FindClass(env, "org/herb/DiffResult");
+  jclass diff_operation_class = (*env)->FindClass(env, "org/herb/DiffOperation");
+  jclass array_list_class = (*env)->FindClass(env, "java/util/ArrayList");
+
+  jmethodID diff_result_constructor = (*env)->GetMethodID(env, diff_result_class, "<init>", "(ZLjava/util/List;)V");
+  jmethodID diff_operation_constructor = (*env)->GetMethodID(env, diff_operation_class, "<init>", "(Ljava/lang/String;[ILjava/lang/Object;Ljava/lang/Object;II)V");
+  jmethodID array_list_constructor = (*env)->GetMethodID(env, array_list_class, "<init>", "()V");
+  jmethodID array_list_add = (*env)->GetMethodID(env, array_list_class, "add", "(Ljava/lang/Object;)Z");
+
+  jobject operations_list = (*env)->NewObject(env, array_list_class, array_list_constructor);
+
+  size_t operation_count = herb_diff_operation_count(diff_result);
+
+  for (size_t index = 0; index < operation_count; index++) {
+    const herb_diff_operation_T* operation = herb_diff_operation_at(diff_result, index);
+
+    jstring type_string = (*env)->NewStringUTF(env, herb_diff_operation_type_to_string(operation->type));
+    jintArray path_array = (*env)->NewIntArray(env, operation->path.depth);
+    jint* path_elements = (*env)->GetIntArrayElements(env, path_array, NULL);
+
+    for (uint16_t path_index = 0; path_index < operation->path.depth; path_index++) {
+      path_elements[path_index] = (jint) operation->path.indices[path_index];
+    }
+
+    (*env)->ReleaseIntArrayElements(env, path_array, path_elements, 0);
+
+    jobject old_node = operation->old_node != NULL ? CreateASTNode(env, (AST_NODE_T*) operation->old_node) : NULL;
+    jobject new_node = operation->new_node != NULL ? CreateASTNode(env, (AST_NODE_T*) operation->new_node) : NULL;
+
+    jobject diff_operation = (*env)->NewObject(
+      env, diff_operation_class, diff_operation_constructor,
+      type_string, path_array, old_node, new_node,
+      (jint) operation->old_index, (jint) operation->new_index
+    );
+
+    (*env)->CallBooleanMethod(env, operations_list, array_list_add, diff_operation);
+  }
+
+  jboolean identical = herb_diff_trees_identical(diff_result) ? JNI_TRUE : JNI_FALSE;
+  jobject result = (*env)->NewObject(env, diff_result_class, diff_result_constructor, identical, operations_list);
+
+  ast_node_free((AST_NODE_T*) old_root, &old_allocator);
+  ast_node_free((AST_NODE_T*) new_root, &new_allocator);
+
+  hb_allocator_destroy(&diff_allocator);
+  hb_allocator_destroy(&old_allocator);
+  hb_allocator_destroy(&new_allocator);
+
+  (*env)->ReleaseStringUTFChars(env, old_source, old_src);
+  (*env)->ReleaseStringUTFChars(env, new_source, new_src);
+
+  return result;
+}
+
 JNIEXPORT jstring JNICALL
 Java_org_herb_Herb_extractHTML(JNIEnv* env, jclass clazz, jstring source) {
   const char* src = (*env)->GetStringUTFChars(env, source, 0);

java/herb_jni.h

@@ -14,6 +14,7 @@ JNIEXPORT jobject JNICALL Java_org_herb_Herb_lex(JNIEnv*, jclass, jstring);
 JNIEXPORT jstring JNICALL Java_org_herb_Herb_extractRuby(JNIEnv*, jclass, jstring, jobject);
 JNIEXPORT jstring JNICALL Java_org_herb_Herb_extractHTML(JNIEnv*, jclass, jstring);
 JNIEXPORT jbyteArray JNICALL Java_org_herb_Herb_parseRuby(JNIEnv*, jclass, jstring);
+JNIEXPORT jobject JNICALL Java_org_herb_Herb_diff(JNIEnv*, jclass, jstring, jstring);
 
 #ifdef __cplusplus
 }

java/org/herb/DiffOperation.java

@@ -0,0 +1,50 @@
+package org.herb;
+
+import java.util.Arrays;
+
+public class DiffOperation {
+  private final String type;
+  private final int[] path;
+  private final Object oldNode;
+  private final Object newNode;
+  private final int oldIndex;
+  private final int newIndex;
+
+  public DiffOperation(String type, int[] path, Object oldNode, Object newNode, int oldIndex, int newIndex) {
+    this.type = type;
+    this.path = path;
+    this.oldNode = oldNode;
+    this.newNode = newNode;
+    this.oldIndex = oldIndex;
+    this.newIndex = newIndex;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public int[] getPath() {
+    return path;
+  }
+
+  public Object getOldNode() {
+    return oldNode;
+  }
+
+  public Object getNewNode() {
+    return newNode;
+  }
+
+  public int getOldIndex() {
+    return oldIndex;
+  }
+
+  public int getNewIndex() {
+    return newIndex;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("DiffOperation{type=%s, path=%s}", type, Arrays.toString(path));
+  }
+}

java/org/herb/DiffResult.java

@@ -0,0 +1,34 @@
+package org.herb;
+
+import java.util.List;
+
+public class DiffResult {
+  private final boolean identical;
+  private final List<DiffOperation> operations;
+
+  public DiffResult(boolean identical, List<DiffOperation> operations) {
+    this.identical = identical;
+    this.operations = operations;
+  }
+
+  public boolean isIdentical() {
+    return identical;
+  }
+
+  public List<DiffOperation> getOperations() {
+    return operations;
+  }
+
+  public int getOperationCount() {
+    return operations.size();
+  }
+
+  @Override
+  public String toString() {
+    if (identical) {
+      return "DiffResult{identical=true}";
+    }
+
+    return String.format("DiffResult{identical=false, operations=%d}", operations.size());
+  }
+}

java/org/herb/Herb.java

@@ -21,6 +21,7 @@ public class Herb {
   public static native String extractRuby(String source, ExtractRubyOptions options);
   public static native String extractHTML(String source);
   public static native byte[] parseRuby(String source);
+  public static native DiffResult diff(String oldSource, String newSource);
 
   public static ParseResult parse(String source) {
     return parse(source, null);

javascript/packages/core/src/backend.ts

@@ -2,12 +2,15 @@ import type { SerializedParseResult } from "./parse-result.js"
 import type { SerializedLexResult } from "./lex-result.js"
 import type { ParseOptions } from "./parser-options.js"
 import type { ExtractRubyOptions } from "./extract-ruby-options.js"
+import type { DiffResult } from "./diff-result.js"
 
 interface LibHerbBackendFunctions {
   lex: (source: string) => SerializedLexResult
 
   parse: (source: string, options?: ParseOptions) => SerializedParseResult
 
+  diff: (oldSource: string, newSource: string) => DiffResult
+
   extractRuby: (source: string, options?: ExtractRubyOptions) => string
   extractHTML: (source: string) => string
 
@@ -21,6 +24,7 @@ export type BackendPromise = () => Promise<LibHerbBackend>
 const expectedFunctions = [
   "parse",
   "lex",
+  "diff",
   "extractRuby",
   "extractHTML",
   "parseRuby",

javascript/packages/core/src/diff-result.ts

@@ -0,0 +1,15 @@
+import type { SerializedNode } from "./nodes/index.js"
+
+export interface DiffOperation {
+  type: string
+  path: number[]
+  oldNode: SerializedNode | null
+  newNode: SerializedNode | null
+  oldIndex: number
+  newIndex: number
+}
+
+export interface DiffResult {
+  identical: boolean
+  operations: DiffOperation[]
+}

javascript/packages/core/src/herb-backend.ts

@@ -11,6 +11,7 @@ import type { LibHerbBackend, BackendPromise } from "./backend.js"
 import type { ParseOptions } from "./parser-options.js"
 import type { ExtractRubyOptions } from "./extract-ruby-options.js"
 import type { PrismParseResult } from "./prism/index.js"
+import type { DiffResult } from "./diff-result.js"
 
 /**
  * The main Herb parser interface, providing methods to lex and parse input.
@@ -117,6 +118,19 @@ export abstract class HerbBackend {
     return deserializePrismParseResult(bytes, source)
   }
 
+  /**
+   * Diffs two source strings and returns the minimal set of AST differences.
+   * @param oldSource - The old source code.
+   * @param newSource - The new source code.
+   * @returns A DiffResult containing the operations.
+   * @throws Error if the backend is not loaded.
+   */
+  diff(oldSource: string, newSource: string): DiffResult {
+    this.ensureBackend()
+
+    return this.backend.diff(ensureString(oldSource), ensureString(newSource))
+  }
+
   /**
    * Extracts HTML from the given source.
    * @param source - The source code to extract HTML from.

javascript/packages/core/src/index.ts

@@ -2,6 +2,7 @@ export * from "./ast-utils.js"
 export * from "./html-constants.js"
 export * from "./html-character-references.js"
 export * from "./backend.js"
+export * from "./diff-result.js"
 export * from "./diagnostic.js"
 export * from "./didyoumean.js"
 export * from "./errors.js"