Skip to content

feat: added option for user-provided buffer, improved type generation #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
osztenkurden wants to merge 8 commits into
base: main
Choose a base branch
from
Open

feat: added option for user-provided buffer, improved type generation #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
615ba03
feat: added option for user-provided buffer, improved type generation
osztenkurden Aug 22, 2025
52ce9e4
fix: naming on the benchmark tests
osztenkurden Aug 22, 2025
c053381
fix: set return type to Buffer instead of Uint8Array
osztenkurden Aug 22, 2025
90ddfea
fix: lint issues
osztenkurden Aug 29, 2025
faee21f
fix: prevent re-using single buffer across all iterations, by pre-all…
osztenkurden Aug 29, 2025
a8d00aa
Merge branch 'main' into main
Brooooooklyn Sep 11, 2025
a5001fd
Merge branch 'main' into main
osztenkurden Sep 11, 2025
6eb4c25
Merge branch 'Brooooooklyn:main' into main
osztenkurden Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion benchmark/bench.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
import { Bench, hrtimeNow } from 'tinybench'
import { compress as legacyCompress, uncompress as legacyUncompress } from 'legacy-snappy'

import { compress, uncompress, compressSync } from '../index.js'
import { compress, uncompress, compressSync, uncompressSync } from '../index.js'
import { fileURLToPath } from 'node:url'

const gzipAsync = promisify(gzip)
Expand All @@ -33,6 +33,25 @@ const SNAPPY_COMPRESSED_FIXTURE = Buffer.from(compressSync(FIXTURE))
const GZIP_FIXTURE = gzipSync(FIXTURE)
const DEFLATED_FIXTURE = deflateSync(FIXTURE)
const BROTLI_COMPRESSED_FIXTURE = brotliCompressSync(FIXTURE)
const MEMORY_POOL = 50_000

const initiateMemoryPool = () => {
const pool = new Array(MEMORY_POOL) as Uint8Array[]
for (let i = 0; i < MEMORY_POOL; i++) {
pool[i] = new Uint8Array(FIXTURE.length)
}

const response = {
currentBufferIndex: 0,
getAvailableBuffer() {
const buffer = pool[response.currentBufferIndex]
response.currentBufferIndex++
return buffer!
},
}

return response
}

const b = new Bench({
now: hrtimeNow,
Expand All @@ -42,6 +61,10 @@ b.add('snappy-compress', () => {
return compress(FIXTURE)
})

b.add('snappy-compress-sync', () => {
return compressSync(FIXTURE)
})

b.add('snappy-v6-compress', () => {
return compressV6(FIXTURE)
})
Expand All @@ -65,10 +88,22 @@ console.table(b.table())
const bUncompress = new Bench({
now: hrtimeNow,
})
const outputPool = initiateMemoryPool() // new Uint8Array(FIXTURE.length)

bUncompress.add('snappy-uncompress', () => {
return uncompress(SNAPPY_COMPRESSED_FIXTURE)
})
bUncompress.add('snappy-alloc-uncompress', () => {
return uncompress(SNAPPY_COMPRESSED_FIXTURE, { output: outputPool.getAvailableBuffer() })
})
bUncompress.add('snappy-sync-uncompress', () => {
return uncompressSync(SNAPPY_COMPRESSED_FIXTURE)
})
const outputPool2 = initiateMemoryPool()

bUncompress.add('snappy-sync-alloc-uncompress', () => {
return uncompressSync(SNAPPY_COMPRESSED_FIXTURE, { output: outputPool2.getAvailableBuffer() })
})

bUncompress.add('snappy-v6-uncompress', () => {
// @ts-expect-error
Expand Down
15 changes: 13 additions & 2 deletions index.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export interface DecOptions {
* see https://www.electronjs.org/blog/v8-memory-cage and https://github.com/electron/electron/issues/35801#issuecomment-1261206333
*/
copyOutputData?: boolean
output?: Uint8Array
}

export interface EncOptions {
Expand All @@ -31,6 +32,16 @@ export interface EncOptions {
copyOutputData?: boolean
}

export declare function uncompress(input: string | Uint8Array, options?: DecOptions | undefined | null, signal?: AbortSignal | undefined | null): Promise<string | Buffer>
export declare function uncompress(input: string | Uint8Array, options?: DecOptions | undefined | null, signal?: AbortSignal | undefined | null): Promise<Buffer>
export declare function uncompress(input: string | Uint8Array, options: { asBuffer: false }): Promise<string>;
export declare function uncompress(input: string | Uint8Array, options: { output: Uint8Array }): Promise<number>;
export declare function uncompress(input: string | Uint8Array, options?: { asBuffer?: true }): Promise<Buffer>;
export declare function uncompress(input: string | Uint8Array, options?: DecOptions): Promise<string | Buffer | number>;


export declare function uncompressSync(input: undefined, options?: DecOptions | undefined | null): Buffer
export declare function uncompressSync(input: string | Uint8Array, options: { asBuffer: false }): string;
export declare function uncompressSync(input: string | Uint8Array, options: { output: Uint8Array }): number;
export declare function uncompressSync(input: string | Uint8Array, options?: { asBuffer?: true }): Buffer;
export declare function uncompressSync(input: string | Uint8Array, options?: DecOptions): string | Buffer | number;

export declare function uncompressSync(input: string | Uint8Array, options?: DecOptions | undefined | null): string | Buffer
108 changes: 75 additions & 33 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub struct DecOptions {
/// for compatibility with electron >= 21 \n
/// see https://www.electronjs.org/blog/v8-memory-cage and https://github.com/electron/electron/issues/35801#issuecomment-1261206333
pub copy_output_data: Option<bool>,
pub output: Option<Uint8Array>,
}

#[napi(object)]
Expand Down Expand Up @@ -74,31 +75,49 @@ pub struct Dec {

#[napi]
impl<'env> ScopedTask<'env> for Dec {
type Output = Vec<u8>;
type JsValue = Either<String, BufferSlice<'env>>;
type Output = Either<Vec<u8>, u32>;
type JsValue = Either3<String, BufferSlice<'env>, u32>;

fn compute(&mut self) -> Result<Self::Output> {
let input_data = match &self.data {
Either::A(ref s) => s.as_bytes(),
Either::B(b) => b.as_ref(),
};

if let Some(ref mut opts) = self.options {
if let Some(ref mut output_buffer) = opts.output {
let decompressed_len = self
.inner

Copilot AI Aug 29, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The use of unsafe to get a mutable reference to the output buffer is potentially dangerous. Consider using safe alternatives or adding proper safety documentation explaining why this unsafe block is necessary and what invariants must be maintained.

Copilot uses AI. Check for mistakes.

@osztenkurden osztenkurden Aug 29, 2025

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Brooooooklyn a question from me - I see no other way than using the unsafe in this small part. Would adding:

          // SAFETY: We know the buffer is valid for the lifetime of this function
          // and we're not extending beyond its bounds

comment be sufficient enough? I might be missing something, as Im pretty new to Rust.

.decompress(input_data, unsafe { output_buffer.as_mut() })
.map_err(|err| Error::new(Status::GenericFailure, format!("{err}")))?;

return Ok(Either::B(decompressed_len as u32));
}
}
self
.inner
.decompress_vec(match self.data {
Either::A(ref s) => s.as_bytes(),
Either::B(ref b) => b.as_ref(),
})
.decompress_vec(input_data)
.map(Either::A)
.map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))
}

fn resolve(&mut self, env: &'env Env, output: Self::Output) -> Result<Self::JsValue> {
let opt_ref = self.options.as_ref();
if opt_ref.and_then(|o| o.as_buffer).unwrap_or(true) {
if opt_ref.and_then(|o| o.copy_output_data).unwrap_or(false) {
BufferSlice::copy_from(env, output).map(Either::B)
} else {
BufferSlice::from_data(env, output).map(Either::B)
match output {
Either::B(length) => Ok(Either3::C(length)),
Either::A(output) => {
let opt_ref = self.options.as_ref();
if opt_ref.and_then(|o| o.as_buffer).unwrap_or(true) {
if opt_ref.and_then(|o| o.copy_output_data).unwrap_or(false) {
BufferSlice::copy_from(env, output).map(Either3::B)
} else {
BufferSlice::from_data(env, output).map(Either3::B)
}
} else {
Ok(Either3::A(String::from_utf8(output).map_err(|e| {
Error::new(Status::GenericFailure, format!("{e}"))
})?))
}
}
} else {
Ok(Either::A(String::from_utf8(output).map_err(|e| {
Error::new(Status::GenericFailure, format!("{e}"))
})?))
}
}
}
Expand Down Expand Up @@ -144,39 +163,62 @@ pub fn compress(
AsyncTask::with_optional_signal(encoder, signal)
}

#[napi]
#[napi(ts_return_type = r#"Buffer
export declare function uncompressSync(input: string | Uint8Array, options: { asBuffer: false }): string;
export declare function uncompressSync(input: string | Uint8Array, options: { output: Uint8Array }): number;
export declare function uncompressSync(input: string | Uint8Array, options?: { asBuffer?: true }): Buffer;
export declare function uncompressSync(input: string | Uint8Array, options?: DecOptions): string | Buffer | number;
"#)]
pub fn uncompress_sync<'env>(
env: &'env Env,
input: Either<String, &'env [u8]>,
#[napi(ts_arg_type = "undefined")] input: Either<String, &'env [u8]>,
options: Option<DecOptions>,
) -> Result<Either<String, BufferSlice<'env>>> {
) -> Result<Either3<String, BufferSlice<'env>, u32>> {
let mut dec = Decoder::new();
let input_data = match input {
Either::A(ref s) => s.as_bytes(),
Either::B(b) => b,
};

let as_buffer = options.as_ref().and_then(|o| o.as_buffer).unwrap_or(true);
let copy_output_data = options
.as_ref()
.and_then(|o| o.copy_output_data)
.unwrap_or(false);

if let Some(mut opts) = options {
if let Some(ref mut output_buffer) = opts.output {
Comment thread
osztenkurden marked this conversation as resolved.
let decompressed_len = dec
.decompress(input_data, unsafe { output_buffer.as_mut() })
.map_err(|err| Error::new(Status::GenericFailure, format!("{err}")))?;

return Ok(Either3::C(decompressed_len as u32));
}
}
dec
.decompress_vec(match input {
Either::A(ref s) => s.as_bytes(),
Either::B(b) => b,
})
.decompress_vec(input_data)
.map_err(|err| Error::new(Status::GenericFailure, format!("{err}")))
.and_then(|output| {
if options.as_ref().and_then(|o| o.as_buffer).unwrap_or(true) {
if options
.as_ref()
.and_then(|o| o.copy_output_data)
.unwrap_or(false)
{
BufferSlice::copy_from(env, output).map(Either::B)
if as_buffer {
if copy_output_data {
BufferSlice::copy_from(env, output).map(Either3::B)
} else {
BufferSlice::from_data(env, output).map(Either::B)
BufferSlice::from_data(env, output).map(Either3::B)
}
} else {
Ok(Either::A(String::from_utf8(output).map_err(|e| {
Ok(Either3::A(String::from_utf8(output).map_err(|e| {
Error::new(Status::GenericFailure, format!("{e}"))
})?))
}
})
}

#[napi]
#[napi(ts_return_type = r#"Promise<Buffer>
export declare function uncompress(input: string | Uint8Array, options: { asBuffer: false }): Promise<string>;
export declare function uncompress(input: string | Uint8Array, options: { output: Uint8Array }): Promise<number>;
export declare function uncompress(input: string | Uint8Array, options?: { asBuffer?: true }): Promise<Buffer>;
export declare function uncompress(input: string | Uint8Array, options?: DecOptions): Promise<string | Buffer | number>;
"#)]
pub fn uncompress(
input: Either<String, Uint8Array>,
options: Option<DecOptions>,
Expand Down