Skip to content

Commit c043852

Browse files
OKUA1OKUA1
authored
Merge pull request #15 from BeastByteAI/js-tar-handling
fix: js tar handling
2 parents fadee06 + 8ce1f67 commit c043852

12 files changed

Lines changed: 828 additions & 603 deletions

File tree

‎.github/workflows/npm_publish.yml‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,38 @@ jobs:
7676
working-directory: src/js
7777
run: pnpm build
7878

79-
- name: Publish
79+
- name: Publish @fnnx-ai/common
8080
working-directory: src/js
81-
run: pnpm -r publish --access public --no-git-checks
81+
run: |
82+
VERSION=$(node -p "require('./packages/common/package.json').version")
83+
if npm view @fnnx-ai/common@$VERSION version 2>/dev/null; then
84+
echo "common@$VERSION already published, skipping"
85+
else
86+
pnpm --filter @fnnx-ai/common publish --access public --no-git-checks
87+
fi
88+
env:
89+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
90+
91+
- name: Publish @fnnx-ai/web
92+
working-directory: src/js
93+
run: |
94+
VERSION=$(node -p "require('./packages/web/package.json').version")
95+
if npm view @fnnx-ai/web@$VERSION version 2>/dev/null; then
96+
echo "web@$VERSION already published, skipping"
97+
else
98+
pnpm --filter @fnnx-ai/web publish --access public --no-git-checks
99+
fi
100+
env:
101+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
102+
103+
- name: Publish @fnnx-ai/node
104+
working-directory: src/js
105+
run: |
106+
VERSION=$(node -p "require('./packages/node/package.json').version")
107+
if npm view @fnnx-ai/node@$VERSION version 2>/dev/null; then
108+
echo "node@$VERSION already published, skipping"
109+
else
110+
pnpm --filter @fnnx-ai/node publish --access public --no-git-checks
111+
fi
82112
env:
83113
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

‎src/js/packages/node/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@fnnx-ai/node",
3-
"version": "0.0.1",
3+
"version": "0.0.2",
44
"type": "module",
55
"main": "dist/index.js",
66
"types": "dist/index.d.ts",
@@ -11,7 +11,8 @@
1111
},
1212
"dependencies": {
1313
"@fnnx-ai/common": "workspace:*",
14-
"onnxruntime-node": "^1.20.1"
14+
"onnxruntime-node": "^1.20.1",
15+
"tar": "^7.0.0"
1516
},
1617
"devDependencies": {
1718
"vitest": "^4.0.18"

‎src/js/packages/node/src/model.ts‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, rmSync, mkdtempSync } from "node:fs";
1+
import { readFileSync, mkdirSync, readdirSync, statSync, rmSync, mkdtempSync } from "node:fs";
22
import { tmpdir } from "node:os";
33
import path from "node:path";
4+
import { Readable } from "node:stream";
5+
import { pipeline } from "node:stream/promises";
6+
import { extract as tarExtract } from "tar";
47
import { interfaces, LocalHandler, DtypesManager, Inputs, Outputs, DynamicAttributes, applyPatches } from "@fnnx-ai/common";
5-
import { TarExtractor } from "./tar.js";
68
import { ONNXOpV1 } from "./ops.js";
79

810
const MANIFEST_PATCH_PATTERN = /^manifest-[^/]+\.patch\.json$/;
@@ -44,16 +46,14 @@ export class Model {
4446
return new Model(modelPath, false);
4547
}
4648
const tempDir = mkdtempSync(path.join(tmpdir(), 'fnnx-'));
47-
const fileBuffer = readFileSync(modelPath);
48-
const arrayBuffer = toArrayBuffer(fileBuffer);
49-
extractToDir(arrayBuffer, tempDir);
49+
await tarExtract({ file: modelPath, C: tempDir });
5050
return new Model(tempDir, true);
5151
}
5252

5353
static async fromBuffer(modelData: ArrayBuffer | Buffer): Promise<Model> {
5454
const tempDir = mkdtempSync(path.join(tmpdir(), 'fnnx-'));
55-
const arrayBuffer = Buffer.isBuffer(modelData) ? toArrayBuffer(modelData) : modelData;
56-
extractToDir(arrayBuffer, tempDir);
55+
const buf = Buffer.isBuffer(modelData) ? modelData : Buffer.from(modelData);
56+
await extractToDir(buf, tempDir);
5757
return new Model(tempDir, true);
5858
}
5959

@@ -135,25 +135,9 @@ export class Model {
135135
}
136136
}
137137

138-
function toArrayBuffer(buf: Buffer): ArrayBuffer {
139-
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
140-
}
141-
142-
function extractToDir(tarBuffer: ArrayBuffer, targetDir: string): void {
143-
const extractor = new TarExtractor(tarBuffer);
144-
const files = extractor.extract();
145-
146-
for (const file of files) {
147-
const targetPath = path.join(targetDir, file.relpath);
148-
if (file.type === 'directory') {
149-
mkdirSync(targetPath, { recursive: true });
150-
} else {
151-
mkdirSync(path.dirname(targetPath), { recursive: true });
152-
if (file.content) {
153-
writeFileSync(targetPath, file.content);
154-
}
155-
}
156-
}
138+
async function extractToDir(tarBuffer: Buffer, targetDir: string): Promise<void> {
139+
const readable = Readable.from(tarBuffer);
140+
await pipeline(readable, tarExtract({ C: targetDir }));
157141
}
158142

159143
function listModelFiles(dir: string): interfaces.TarFileContent[] {

‎src/js/packages/node/src/tar.ts‎

Lines changed: 0 additions & 136 deletions
This file was deleted.

‎src/js/packages/node/tests/model.test.ts‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,20 @@
11
import { describe, it, expect, beforeAll, afterAll } from "vitest";
2-
import { readFileSync, existsSync, mkdtempSync, mkdirSync, writeFileSync, cpSync } from "node:fs";
2+
import { readFileSync, mkdtempSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import path from "node:path";
55
import { fileURLToPath } from "node:url";
6+
import { extract as tarExtract } from "tar";
67
import { Model } from "../src/model";
78
import { NDArray } from "@fnnx-ai/common";
8-
import { TarExtractor } from "../src/tar";
99

1010
const __filename = fileURLToPath(import.meta.url);
1111
const __dirname = path.dirname(__filename);
1212

1313
const MODEL_PATH = path.resolve(__dirname, "../../../../python/tests/models/onnx_pipeline.fnnx.tar");
1414

15-
function extractModelToDir(): string {
15+
async function extractModelToDir(): Promise<string> {
1616
const tempDir = mkdtempSync(path.join(tmpdir(), 'fnnx-test-'));
17-
const buffer = readFileSync(MODEL_PATH);
18-
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
19-
const extractor = new TarExtractor(arrayBuffer);
20-
const files = extractor.extract();
21-
22-
for (const file of files) {
23-
const targetPath = path.join(tempDir, file.relpath);
24-
if (file.type === 'directory') {
25-
mkdirSync(targetPath, { recursive: true });
26-
} else {
27-
mkdirSync(path.dirname(targetPath), { recursive: true });
28-
if (file.content) {
29-
writeFileSync(targetPath, file.content);
30-
}
31-
}
32-
}
17+
await tarExtract({ file: MODEL_PATH, C: tempDir });
3318
return tempDir;
3419
}
3520

@@ -56,8 +41,8 @@ describe("Model", () => {
5641
describe("loading from directory", () => {
5742
let modelDir: string;
5843

59-
beforeAll(() => {
60-
modelDir = extractModelToDir();
44+
beforeAll(async () => {
45+
modelDir = await extractModelToDir();
6146
});
6247

6348
it("should load from directory path", async () => {

‎src/js/packages/node/tests/reader.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import path from "node:path";
55
import { Model } from "../src/model";
6-
import { TarExtractor } from "../src/tar";
76

87
function createTarEntry(fileName: string, content: string): Uint8Array {
98
const contentBytes = new TextEncoder().encode(content);

0 commit comments

Comments
 (0)