Refactor buildSrc and enhance generated language bindings and tests

README.md

@@ -65,6 +65,24 @@ class Main {
 - x86_64-linux
 - aarch64-linux
 
+# Developers: How to Add a Parser
+
+To add a new language parser to this project, we provide a code generation task that handles most of the boilerplate. This is also how you can add an "unofficial" or community parser.
+
+1. **Generate the subproject:**
+   Run the `gen` task, providing the language name (in this example, Kotlin), its version, and the URL to its source code zip file.
+   ```bash
+   ./gradlew gen --parser-name=kotlin --parser-version=0.3.8 --parser-zip=https://github.com/fwcd/tree-sitter-kotlin/archive/refs/tags/0.3.8.zip
+   ```
+   This will create a new directory `tree-sitter-kotlin` with the correct `build.gradle`, `gradle.properties`, JNI bindings, and Java class extending `TSLanguage`. Finally, an entry of `include 'tree-sitter-kotlin'` will be inserted into `settings.gradle`.
+
+2. **Build native modules and test:**
+   Our build system automatically uses Zig to cross-compile the native shared libraries for the new parser. You can trigger the download, native compilation, and tests:
+   ```bash
+   ./gradlew :tree-sitter-kotlin:buildNative
+   ./gradlew :tree-sitter-kotlin:test
+   ```
+
 # Built-in official parsers
 | Name                            | Version                                                                                                 |
 |---------------------------------|---------------------------------------------------------------------------------------------------------|

buildSrc/build.gradle

@@ -1,7 +1,18 @@
+plugins {
+    id 'groovy-gradle-plugin'
+}
+
 repositories {
     mavenCentral()
 }
 dependencies {
-    implementation 'org.apache.commons:commons-compress:1.27.1'
-    implementation 'org.tukaani:xz:1.10'
+    implementation libs.commons.compress
+    implementation libs.xz
+    testImplementation platform(libs.junit.bom)
+    testImplementation libs.junit.jupiter
+    testRuntimeOnly "org.junit.platform:junit-platform-launcher"
+}
+
+test {
+    useJUnitPlatform()
 }

buildSrc/settings.gradle

@@ -0,0 +1,7 @@
+dependencyResolutionManagement {
+    versionCatalogs {
+        libs {
+            from(files("../gradle/libs.versions.toml"))
+        }
+    }
+}

buildSrc/src/main/groovy/org/treesitter/build/GenTask.groovy

@@ -31,22 +31,24 @@ package org.treesitter;
 
 import org.treesitter.utils.NativeUtils;
 
-public class $className implements TSLanguage {
+public class $className extends TSLanguage {
 
     static {
         NativeUtils.loadLib("lib/tree-sitter-$libShortName");
     }
     private native static long tree_sitter_$idName();
 
-    private final long ptr;
-
     public $className() {
-        ptr = tree_sitter_$idName();
+        super(tree_sitter_$idName());
+    }
+
+    private $className(long ptr) {
+        super(ptr);
     }
 
     @Override
-    public long getPtr() {
-        return ptr;
+    public TSLanguage copy() {
+        return new $className(copyPtr());
     }
 }
 """
@@ -63,11 +65,14 @@ public class $className implements TSLanguage {
 package org.treesitter;
 
 import org.junit.jupiter.api.Test;
+import org.treesitter.tests.CorpusTest;
+
+import java.io.IOException;
 
 class TreeSitter${capitalized}Test {
     @Test
-    void init() {
-        new TreeSitter$capitalized();
+    void corpusTest() throws IOException {
+        CorpusTest.runAllTestsInDefaultFolder(new TreeSitter$capitalized(), "$libShortName");
     }
 }
 """
@@ -86,161 +91,10 @@ class TreeSitter${capitalized}Test {
 
 
     static void genBuildGradle(File projectDir, String libShortName, String url){
-        def capitalized = capitalizedLibName(libShortName)
         def gradleFile = new File(projectDir, "build.gradle")
         def content = """
-
-import org.treesitter.build.Utils
-
-plugins {
-    id 'java'
-    id 'signing'
-    id 'maven-publish'
-}
-
-group = 'io.github.bonede'
-version = libVersion
-
-repositories {
-    mavenCentral()
-}
-
-dependencies {
-    testImplementation platform(libs.junit.bom)
-    testImplementation 'org.junit.jupiter:junit-jupiter'
-    implementation project(":tree-sitter")
-}
-
-test {
-    useJUnitPlatform()
-}
-def libName = "tree-sitter-$libShortName"
-
-
-java {
-    withJavadocJar()
-    withSourcesJar()
-}
-
-publishing {
-    repositories {
-        maven {
-            name = "MavenCentral"
-            def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/"
-            def snapshotsRepoUrl =  "https://s01.oss.sonatype.org/content/repositories/snapshots/"
-            credentials {
-                username = ossrhUsername
-                password = ossrhPassword
-            }
-            url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
-        }
-    }
-    publications {
-        maven(MavenPublication) {
-            from components.java
-            pom {
-                name = libName
-                url = 'https://github.com/bonede/tree-sitter-ng'
-                description = "Next generation Tree Sitter Java binding"
-                licenses {
-                    license {
-                        name = 'MIT'
-                    }
-                }
-                scm {
-                    connection = 'scm:git:https://github.com/bonede/tree-sitter-ng.git'
-                    developerConnection = 'scm:git:https://github.com/bonede/tree-sitter-ng.git'
-                    url = 'https://github.com/bonede/tree-sitter-ng'
-                }
-                developers {
-                    developer {
-                        id = 'bonede'
-                        name = 'Wang Liang'
-                        email = 'bonede@qq.com'
-                    }
-                }
-            }
-        }
-    }
-}
-
-
-signing {
-    sign configurations.archives
-    sign publishing.publications
-}
-tasks.register('downloadSource') {
-    group = "build setup"
-    description = "Download parser source"
-    def zipUrl = "$url"
-    def downloadDir = Utils.libDownloadDir(project, libName)
-    def zip = Utils.libZipFile(project, libName, libVersion)
-    def parserCFile = Utils.libParserCFile(project, libName, libVersion)
-    inputs.files(layout.projectDirectory.file("gradle.properties"))
-    outputs.files(parserCFile)
-    doLast {
-        download.run {
-            src zipUrl
-            dest zip
-            overwrite false
-        }
-        copy {
-            from zipTree(zip)
-            into downloadDir
-        }
-    }
-
-}
-
-tasks.register("buildNative") {
-    group = "build"
-    description = "Build parser native modules"
-    dependsOn "downloadSource", rootProject.bootstrap
-    def jniSrcDir = Utils.jniSrcDir(project)
-    def outDir = Utils.jniOutDir(project)
-    def jniCFile = Utils.jniCFile(project, "org_treesitter_TreeSitter${capitalized}.c")
-    def parserCFile = Utils.libParserCFile(project, libName, libVersion)
-    def scannerCFile = Utils.libScannerCFile(project, libName, libVersion)
-    def libSrcDir = Utils.libSrcDir(project, libName, libVersion)
-    def jniInclude = Utils.jniIncludeDir(project)
-
-    def targets = Utils.treeSitterTargets(project)
-    def outputFiles = targets.collect()
-            { t -> Utils.jniOutFile(project, t, libName)}
-    def srcFiles = project.fileTree(libSrcDir) {
-        include(Utils.libFiles())
-    }.toList()
-    outputs.files(outputFiles)
-    def inputFiles = srcFiles + [parserCFile, rootProject.layout.projectDirectory.file("gradle.properties")]
-    inputs.files(inputFiles)
-    doLast{
-        mkdir(outDir)
-        targets.each {target ->
-            def jniMdInclude = Utils.jniMdInclude(project, target)
-            def jniOutFile = Utils.jniOutFile(project, target, libName)
-            def files = project.fileTree(libSrcDir) {
-                include(Utils.libFiles())
-            }.toList()
-            def cmd = [
-                        rootProject.downloadZig.zigExe, "c++",
-                        "-g0",
-                        "-shared",
-                        "-target", target,
-                        "-I", libSrcDir,
-                        "-I", jniInclude,
-                        "-I", jniMdInclude,
-                        "-o", jniOutFile,
-                        jniCFile,
-            ]
-
-            cmd.addAll(files)
-            exec{
-                workingDir jniSrcDir
-                commandLine(cmd)
-            }
-        }
-        Utils.removeWindowsDebugFiles(project)
-    }
+tasks.named('downloadSource') {
+    url = "$url"
 }
 """
         try (OutputStream outputStream = new FileOutputStream(gradleFile)){
@@ -290,7 +144,7 @@ JNIEXPORT jlong JNICALL Java_org_treesitter_TreeSitter${capitalized}_tree_1sitte
         }
         if(shouldUpdate){
             try(OutputStream outputStream = new FileOutputStream(settingsFile, true)){
-                outputStream.withPrintWriter {writer -> writer.println(projectLine)}
+                outputStream.withPrintWriter {writer -> writer.println(System.lineSeparator() + projectLine)}
             }
         }
     }

buildSrc/src/test/groovy/org/treesitter/build/GenTaskTest.groovy

@@ -0,0 +1,97 @@
+package org.treesitter.build
+
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import static org.junit.jupiter.api.Assertions.*
+
+class GenTaskTest {
+
+    @TempDir
+    File tempDir
+
+    @Test
+    void "should generate java file with TSLanguage extension and copy method"() {
+        // Arrange
+        String libName = "json"
+
+        // Act
+        GenTask.genJavaFile(tempDir, libName)
+
+        // Assert
+        File javaFile = new File(tempDir, "src/main/java/org/treesitter/TreeSitterJson.java")
+        assertTrue(javaFile.exists(), "Java file should be generated")
+
+        String content = javaFile.text
+        assertTrue(content.contains("public class TreeSitterJson extends TSLanguage"), "Should extend TSLanguage")
+        assertTrue(content.contains("NativeUtils.loadLib(\"lib/tree-sitter-json\")"), "Should load correct library")
+        assertTrue(content.contains("@Override"), "Should have Override annotation")
+        assertTrue(content.contains("public TSLanguage copy()"), "Should implement copy method")
+        assertTrue(content.contains("return new TreeSitterJson(copyPtr())"), "Should call copyPtr")
+    }
+
+    @Test
+    void "should generate java test file with CorpusTest call"() {
+        // Arrange
+        String libName = "json"
+
+        // Act
+        GenTask.genJavaTestFile(tempDir, libName)
+
+        // Assert
+        File testFile = new File(tempDir, "src/test/java/org/treesitter/TreeSitterJsonTest.java")
+        assertTrue(testFile.exists(), "Test file should be generated")
+
+        String content = testFile.text
+        assertTrue(content.contains("import org.treesitter.tests.CorpusTest;"), "Should import CorpusTest")
+        assertTrue(content.contains("CorpusTest.runAllTestsInDefaultFolder(new TreeSitterJson(), \"json\");"), "Should call runAllTestsInDefaultFolder")
+    }
+
+    @Test
+    void "should generate build gradle file with downloadSource task"() {
+        // Arrange
+        String libName = "json"
+        String url = "https://example.com/tree-sitter-json.zip"
+
+        // Act
+        GenTask.genBuildGradle(tempDir, libName, url)
+
+        // Assert
+        File gradleFile = new File(tempDir, "build.gradle")
+        assertTrue(gradleFile.exists(), "build.gradle should be generated")
+
+        String content = gradleFile.text
+        assertTrue(content.contains("tasks.named('downloadSource')"), "Should configure downloadSource task")
+        assertTrue(content.contains("url = \"$url\""), "Should contain the correct URL")
+    }
+
+    @Test
+    void "should generate properties file with version"() {
+        // Arrange
+        String version = "0.20.0"
+
+        // Act
+        GenTask.genProperties(tempDir, version)
+
+        // Assert
+        File propsFile = new File(tempDir, "gradle.properties")
+        assertTrue(propsFile.exists(), "gradle.properties should be generated")
+        assertEquals("libVersion=0.20.0", propsFile.text.trim())
+    }
+
+    @Test
+    void "should generate JNI C file with correct method mapping"() {
+        // Arrange
+        String libName = "html" // 'html' -> 'tree_sitter_html'
+
+        // Act
+        GenTask.genJniCFile(tempDir, libName)
+
+        // Assert
+        File cFile = new File(tempDir, "src/main/c/org_treesitter_TreeSitterHtml.c")
+        assertTrue(cFile.exists(), "C file should be generated")
+
+        String content = cFile.text
+        assertTrue(content.contains("Java_org_treesitter_TreeSitterHtml_tree_1sitter_1html"), "Should handle underscore escaping for JNI")
+        assertTrue(content.contains("return (jlong) tree_sitter_html();"), "Should call native symbol")
+    }
+}

gradle/libs.versions.toml

@@ -1,10 +1,15 @@
 [versions]
+commons-compress = "1.27.1"
 download = "5.6.0"
 junit = "5.11.4"
 nexus-staging-version = "0.30.0"
+xz = "1.10"
 
 [libraries]
+commons-compress = { module = "org.apache.commons:commons-compress", version.ref = "commons-compress" }
 junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" }
+junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
+xz = { module = "org.tukaani:xz", version.ref = "xz" }
 
 [plugins]
 download = { id = "de.undercouch.download", version.ref = "download" }