From f8fbd08f1eff164b5fc99ebb509d77d5245f5d40 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Thu, 12 Mar 2026 19:23:37 +0200 Subject: [PATCH 1/5] refactor: remove redundant gradle template code in GenTask --- .../org/treesitter/build/GenTask.groovy | 176 ++---------------- 1 file changed, 15 insertions(+), 161 deletions(-) diff --git a/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy b/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy index ab77f4bb..79766f9a 100644 --- a/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy +++ b/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy @@ -31,22 +31,24 @@ package org.treesitter; import org.treesitter.utils.NativeUtils; -public class $className implements TSLanguage { +public class $className extends TSLanguage { static { NativeUtils.loadLib("lib/tree-sitter-$libShortName"); } private native static long tree_sitter_$idName(); - private final long ptr; - public $className() { - ptr = tree_sitter_$idName(); + super(tree_sitter_$idName()); + } + + private $className(long ptr) { + super(ptr); } @Override - public long getPtr() { - return ptr; + public TSLanguage copy() { + return new $className(copyPtr()); } } """ @@ -63,11 +65,14 @@ public class $className implements TSLanguage { package org.treesitter; import org.junit.jupiter.api.Test; +import org.treesitter.tests.CorpusTest; + +import java.io.IOException; class TreeSitter${capitalized}Test { @Test - void init() { - new TreeSitter$capitalized(); + void corpusTest() throws IOException { + CorpusTest.runAllTestsInDefaultFolder(new TreeSitter$capitalized(), "$libShortName"); } } """ @@ -86,161 +91,10 @@ class TreeSitter${capitalized}Test { static void genBuildGradle(File projectDir, String libShortName, String url){ - def capitalized = capitalizedLibName(libShortName) def gradleFile = new File(projectDir, "build.gradle") def content = """ - -import org.treesitter.build.Utils - -plugins { - id 'java' - id 'signing' - id 'maven-publish' -} - -group = 'io.github.bonede' -version = libVersion - -repositories { - mavenCentral() -} - -dependencies { - testImplementation platform(libs.junit.bom) - testImplementation 'org.junit.jupiter:junit-jupiter' - implementation project(":tree-sitter") -} - -test { - useJUnitPlatform() -} -def libName = "tree-sitter-$libShortName" - - -java { - withJavadocJar() - withSourcesJar() -} - -publishing { - repositories { - maven { - name = "MavenCentral" - def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" - def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/" - credentials { - username = ossrhUsername - password = ossrhPassword - } - url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl - } - } - publications { - maven(MavenPublication) { - from components.java - pom { - name = libName - url = 'https://github.com/bonede/tree-sitter-ng' - description = "Next generation Tree Sitter Java binding" - licenses { - license { - name = 'MIT' - } - } - scm { - connection = 'scm:git:https://github.com/bonede/tree-sitter-ng.git' - developerConnection = 'scm:git:https://github.com/bonede/tree-sitter-ng.git' - url = 'https://github.com/bonede/tree-sitter-ng' - } - developers { - developer { - id = 'bonede' - name = 'Wang Liang' - email = 'bonede@qq.com' - } - } - } - } - } -} - - -signing { - sign configurations.archives - sign publishing.publications -} -tasks.register('downloadSource') { - group = "build setup" - description = "Download parser source" - def zipUrl = "$url" - def downloadDir = Utils.libDownloadDir(project, libName) - def zip = Utils.libZipFile(project, libName, libVersion) - def parserCFile = Utils.libParserCFile(project, libName, libVersion) - inputs.files(layout.projectDirectory.file("gradle.properties")) - outputs.files(parserCFile) - doLast { - download.run { - src zipUrl - dest zip - overwrite false - } - copy { - from zipTree(zip) - into downloadDir - } - } - -} - -tasks.register("buildNative") { - group = "build" - description = "Build parser native modules" - dependsOn "downloadSource", rootProject.bootstrap - def jniSrcDir = Utils.jniSrcDir(project) - def outDir = Utils.jniOutDir(project) - def jniCFile = Utils.jniCFile(project, "org_treesitter_TreeSitter${capitalized}.c") - def parserCFile = Utils.libParserCFile(project, libName, libVersion) - def scannerCFile = Utils.libScannerCFile(project, libName, libVersion) - def libSrcDir = Utils.libSrcDir(project, libName, libVersion) - def jniInclude = Utils.jniIncludeDir(project) - - def targets = Utils.treeSitterTargets(project) - def outputFiles = targets.collect() - { t -> Utils.jniOutFile(project, t, libName)} - def srcFiles = project.fileTree(libSrcDir) { - include(Utils.libFiles()) - }.toList() - outputs.files(outputFiles) - def inputFiles = srcFiles + [parserCFile, rootProject.layout.projectDirectory.file("gradle.properties")] - inputs.files(inputFiles) - doLast{ - mkdir(outDir) - targets.each {target -> - def jniMdInclude = Utils.jniMdInclude(project, target) - def jniOutFile = Utils.jniOutFile(project, target, libName) - def files = project.fileTree(libSrcDir) { - include(Utils.libFiles()) - }.toList() - def cmd = [ - rootProject.downloadZig.zigExe, "c++", - "-g0", - "-shared", - "-target", target, - "-I", libSrcDir, - "-I", jniInclude, - "-I", jniMdInclude, - "-o", jniOutFile, - jniCFile, - ] - - cmd.addAll(files) - exec{ - workingDir jniSrcDir - commandLine(cmd) - } - } - Utils.removeWindowsDebugFiles(project) - } +tasks.named('downloadSource') { + url = "$url" } """ try (OutputStream outputStream = new FileOutputStream(gradleFile)){ From 633c134bf24288b7d31b4cca94ff635ef483faa2 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Thu, 12 Mar 2026 19:23:51 +0200 Subject: [PATCH 2/5] test: add unit tests for GenTaskTest --- .../org/treesitter/build/GenTaskTest.groovy | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 buildSrc/src/test/groovy/org/treesitter/build/GenTaskTest.groovy diff --git a/buildSrc/src/test/groovy/org/treesitter/build/GenTaskTest.groovy b/buildSrc/src/test/groovy/org/treesitter/build/GenTaskTest.groovy new file mode 100644 index 00000000..d9faebb5 --- /dev/null +++ b/buildSrc/src/test/groovy/org/treesitter/build/GenTaskTest.groovy @@ -0,0 +1,97 @@ +package org.treesitter.build + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import static org.junit.jupiter.api.Assertions.* + +class GenTaskTest { + + @TempDir + File tempDir + + @Test + void "should generate java file with TSLanguage extension and copy method"() { + // Arrange + String libName = "json" + + // Act + GenTask.genJavaFile(tempDir, libName) + + // Assert + File javaFile = new File(tempDir, "src/main/java/org/treesitter/TreeSitterJson.java") + assertTrue(javaFile.exists(), "Java file should be generated") + + String content = javaFile.text + assertTrue(content.contains("public class TreeSitterJson extends TSLanguage"), "Should extend TSLanguage") + assertTrue(content.contains("NativeUtils.loadLib(\"lib/tree-sitter-json\")"), "Should load correct library") + assertTrue(content.contains("@Override"), "Should have Override annotation") + assertTrue(content.contains("public TSLanguage copy()"), "Should implement copy method") + assertTrue(content.contains("return new TreeSitterJson(copyPtr())"), "Should call copyPtr") + } + + @Test + void "should generate java test file with CorpusTest call"() { + // Arrange + String libName = "json" + + // Act + GenTask.genJavaTestFile(tempDir, libName) + + // Assert + File testFile = new File(tempDir, "src/test/java/org/treesitter/TreeSitterJsonTest.java") + assertTrue(testFile.exists(), "Test file should be generated") + + String content = testFile.text + assertTrue(content.contains("import org.treesitter.tests.CorpusTest;"), "Should import CorpusTest") + assertTrue(content.contains("CorpusTest.runAllTestsInDefaultFolder(new TreeSitterJson(), \"json\");"), "Should call runAllTestsInDefaultFolder") + } + + @Test + void "should generate build gradle file with downloadSource task"() { + // Arrange + String libName = "json" + String url = "https://example.com/tree-sitter-json.zip" + + // Act + GenTask.genBuildGradle(tempDir, libName, url) + + // Assert + File gradleFile = new File(tempDir, "build.gradle") + assertTrue(gradleFile.exists(), "build.gradle should be generated") + + String content = gradleFile.text + assertTrue(content.contains("tasks.named('downloadSource')"), "Should configure downloadSource task") + assertTrue(content.contains("url = \"$url\""), "Should contain the correct URL") + } + + @Test + void "should generate properties file with version"() { + // Arrange + String version = "0.20.0" + + // Act + GenTask.genProperties(tempDir, version) + + // Assert + File propsFile = new File(tempDir, "gradle.properties") + assertTrue(propsFile.exists(), "gradle.properties should be generated") + assertEquals("libVersion=0.20.0", propsFile.text.trim()) + } + + @Test + void "should generate JNI C file with correct method mapping"() { + // Arrange + String libName = "html" // 'html' -> 'tree_sitter_html' + + // Act + GenTask.genJniCFile(tempDir, libName) + + // Assert + File cFile = new File(tempDir, "src/main/c/org_treesitter_TreeSitterHtml.c") + assertTrue(cFile.exists(), "C file should be generated") + + String content = cFile.text + assertTrue(content.contains("Java_org_treesitter_TreeSitterHtml_tree_1sitter_1html"), "Should handle underscore escaping for JNI") + assertTrue(content.contains("return (jlong) tree_sitter_html();"), "Should call native symbol") + } +} From 78704cca10823c2d6bbf4d99fb3ae7f2ee9de3ec Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Thu, 12 Mar 2026 19:37:45 +0200 Subject: [PATCH 3/5] refactor: use version catalog in buildSrc - Migrate `buildSrc` dependencies to use `libs.versions.toml`. - Add `groovy-gradle-plugin` to `buildSrc`. - Configure JUnit 5 testing support for `buildSrc` logic. --- buildSrc/build.gradle | 15 +++++++++++++-- buildSrc/settings.gradle | 7 +++++++ gradle/libs.versions.toml | 5 +++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 buildSrc/settings.gradle diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index ef3c6beb..7c26b3cb 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -1,7 +1,18 @@ +plugins { + id 'groovy-gradle-plugin' +} + repositories { mavenCentral() } dependencies { - implementation 'org.apache.commons:commons-compress:1.27.1' - implementation 'org.tukaani:xz:1.10' + implementation libs.commons.compress + implementation libs.xz + testImplementation platform(libs.junit.bom) + testImplementation libs.junit.jupiter + testRuntimeOnly "org.junit.platform:junit-platform-launcher" +} + +test { + useJUnitPlatform() } \ No newline at end of file diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle new file mode 100644 index 00000000..6f31e6ef --- /dev/null +++ b/buildSrc/settings.gradle @@ -0,0 +1,7 @@ +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../gradle/libs.versions.toml")) + } + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a94d3baf..61d44375 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,10 +1,15 @@ [versions] +commons-compress = "1.27.1" download = "5.6.0" junit = "5.11.4" nexus-staging-version = "0.30.0" +xz = "1.10" [libraries] +commons-compress = { module = "org.apache.commons:commons-compress", version.ref = "commons-compress" } junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" } +junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } +xz = { module = "org.tukaani:xz", version.ref = "xz" } [plugins] download = { id = "de.undercouch.download", version.ref = "download" } From 455591cf9f507e37a3b9d824c259819fb216afef Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 16 Mar 2026 10:53:40 +0200 Subject: [PATCH 4/5] docs: add instructions for adding new parsers to README --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 909f9460..82cbf066 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,29 @@ class Main { - x86_64-linux - aarch64-linux +# Developers: How to Add a Parser + +To add a new language parser to this project, we provide a code generation task that handles most of the boilerplate. + +1. **Generate the subproject:** + Run the `gen` task, providing the language name, its version, and the URL to its source code zip file. + ```bash + ./gradlew gen -PlibName=some_lang -PlibVersion=0.20.0 -Purl=https://github.com/tree-sitter/tree-sitter-some_lang/archive/refs/tags/v0.20.0.zip + ``` + This will create a new directory `tree-sitter-some_lang` with the correct `build.gradle`, `gradle.properties`, JNI bindings, and Java class extending `TSLanguage`. + +2. **Include the subproject:** + Add the new module to `settings.gradle`: + ```groovy + include 'tree-sitter-some_lang' + ``` + +3. **Build native modules and test:** + Our build system automatically uses Zig to cross-compile the native shared libraries for the new parser. You can trigger the download, native compilation, and tests all at once: + ```bash + ./gradlew :tree-sitter-some_lang:test + ``` + # Built-in official parsers | Name | Version | |---------------------------------|---------------------------------------------------------------------------------------------------------| From d83c399a3a2ff3743d41041561ca6fc62f7df2f6 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 16 Mar 2026 11:29:39 +0200 Subject: [PATCH 5/5] Documented on how to add a parser + tested everything --- README.md | 21 +++++++------------ .../org/treesitter/build/GenTask.groovy | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 82cbf066..cd0ee939 100644 --- a/README.md +++ b/README.md @@ -67,25 +67,20 @@ class Main { # Developers: How to Add a Parser -To add a new language parser to this project, we provide a code generation task that handles most of the boilerplate. +To add a new language parser to this project, we provide a code generation task that handles most of the boilerplate. This is also how you can add an "unofficial" or community parser. 1. **Generate the subproject:** - Run the `gen` task, providing the language name, its version, and the URL to its source code zip file. + Run the `gen` task, providing the language name (in this example, Kotlin), its version, and the URL to its source code zip file. ```bash - ./gradlew gen -PlibName=some_lang -PlibVersion=0.20.0 -Purl=https://github.com/tree-sitter/tree-sitter-some_lang/archive/refs/tags/v0.20.0.zip + ./gradlew gen --parser-name=kotlin --parser-version=0.3.8 --parser-zip=https://github.com/fwcd/tree-sitter-kotlin/archive/refs/tags/0.3.8.zip ``` - This will create a new directory `tree-sitter-some_lang` with the correct `build.gradle`, `gradle.properties`, JNI bindings, and Java class extending `TSLanguage`. + This will create a new directory `tree-sitter-kotlin` with the correct `build.gradle`, `gradle.properties`, JNI bindings, and Java class extending `TSLanguage`. Finally, an entry of `include 'tree-sitter-kotlin'` will be inserted into `settings.gradle`. -2. **Include the subproject:** - Add the new module to `settings.gradle`: - ```groovy - include 'tree-sitter-some_lang' - ``` - -3. **Build native modules and test:** - Our build system automatically uses Zig to cross-compile the native shared libraries for the new parser. You can trigger the download, native compilation, and tests all at once: +2. **Build native modules and test:** + Our build system automatically uses Zig to cross-compile the native shared libraries for the new parser. You can trigger the download, native compilation, and tests: ```bash - ./gradlew :tree-sitter-some_lang:test + ./gradlew :tree-sitter-kotlin:buildNative + ./gradlew :tree-sitter-kotlin:test ``` # Built-in official parsers diff --git a/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy b/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy index 79766f9a..cb2bfee0 100644 --- a/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy +++ b/buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy @@ -144,7 +144,7 @@ JNIEXPORT jlong JNICALL Java_org_treesitter_TreeSitter${capitalized}_tree_1sitte } if(shouldUpdate){ try(OutputStream outputStream = new FileOutputStream(settingsFile, true)){ - outputStream.withPrintWriter {writer -> writer.println(projectLine)} + outputStream.withPrintWriter {writer -> writer.println(System.lineSeparator() + projectLine)} } } }