fix(thicktoken): rm slice ordering (#672)

Author: franklevasseur

thicktoken/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bpinternal/thicktoken",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "Tiktoken but thicker; A bundled tokenizer with added helper functions",
   "repository": {
     "url": "https://github.com/botpress/packages"

thicktoken/src/smart-slice.test.ts

@@ -31,9 +31,13 @@ test('SmartSlice handles slices of zero elements', () => {
   const indices1 = [...slice1]
   expect(indices1).toEqual([])
 
-  const slice2 = new SmartSlice([[-5, 5]], 10)
-  const indices2 = [...slice2]
+  const slices2 = new SmartSlice([[9, 1]], 10)
+  const indices2 = [...slices2]
   expect(indices2).toEqual([])
+
+  const slice3 = new SmartSlice([[-5, 5]], 10)
+  const indices3 = [...slice3]
+  expect(indices3).toEqual([])
 })
 
 test('SmartSlice handles overlapping slices', () => {

thicktoken/src/smart-slice.ts

@@ -5,7 +5,8 @@ export class SmartSlice {
 
   public constructor(slices: Slice[], max: number) {
     const clampedSlices: Slice[] = slices.map((s) => this._clampSlice(s, max))
-    const sortedSlices: Slice[] = clampedSlices.sort((a, b) => a[0] - b[0])
+    const nonEmptySlices: Slice[] = clampedSlices.filter(([start, end]) => start < end)
+    const sortedSlices: Slice[] = nonEmptySlices.sort((a, b) => a[0] - b[0])
 
     const merged: Slice[] = []
     for (const [start, end] of sortedSlices) {
@@ -29,13 +30,6 @@ export class SmartSlice {
     start = start < 0 ? max + start : start
     end = end < 0 ? max + end : end
 
-    // order slice
-    if (end < start) {
-      const tmp = start
-      start = end
-      end = tmp
-    }
-
     // clamp slice
     const clampedStart = Math.max(0, Math.min(start, max))
     const clampedEnd = Math.max(0, Math.min(end, max))