Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ Use `MemoryStreamUploadHandler` for complete file uploads (uploads when recordin

## Audio Format

Specify the audio format using MIME type:
Specify the audio format using MIME type and optionally the sample rate in Hz (e.g. 16000 for speech, 48000 for high-fidelity). When `SampleRate` is null, the browser uses its default (typically 48000 Hz). Common values: 8000, 11025, 16000, 22050, 24000, 32000, 44100, 48000; other values may be used and the browser will pick the closest supported rate.

```csharp demo-below
public class AudioFormatDemo : ViewBase
Expand All @@ -95,6 +95,7 @@ public class AudioFormatDemo : ViewBase
return Layout.Vertical()
| new AudioInput(upload.Value, "Record WebM", "Recording WebM...")
.MimeType("audio/webm")
.SampleRate(24000)
| (audioFile.Value != null
? Text.P($"Format: {audioFile.Value.ContentType}, Size: {StringHelper.FormatBytes(audioFile.Value.Length)}").Small()
: null);
Expand Down
36 changes: 35 additions & 1 deletion src/Ivy.Samples.Shared/Apps/Widgets/Inputs/AudioInputApp.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ public class AudioInputApp() : SampleBase
| (Layout.Horizontal().Gap(4)
| new Card(new AudioInputBasic()).Title("Basic")
| new Card(new AudioInputChunkedUpload()).Title("Chunked Upload")
| new Card(new AudioInputDisabledState()).Title("Disabled State"))
| new Card(new AudioInputDisabledState()).Title("Disabled State")
| new Card(new AudioInputSampleRate(24000)).Title("24 kHz (speech)"))
| Text.H2("Sizes")
| CreateSizesSection(dummyUpload.Value);
}
Expand Down Expand Up @@ -113,6 +114,39 @@ public class AudioInputChunkedUpload : ViewBase
}
}

public class AudioInputSampleRate : ViewBase
{
private readonly int? _sampleRate;

public AudioInputSampleRate(int? sampleRate)
{
_sampleRate = sampleRate;
}

public override object? Build()
{
var audioFile = UseState<FileUpload<byte[]>?>();
var upload = UseUpload(
MemoryStreamUploadHandler.Create(audioFile),
defaultContentType: "audio/webm"
);

var label = _sampleRate.HasValue ? $"Record at {_sampleRate} Hz" : "Record (browser default)";
var input = new AudioInput(upload.Value, label, "Recording...");
if (_sampleRate.HasValue)
input = input.SampleRate(_sampleRate.Value);

return Layout.Vertical().Gap(4)
| Text.P(_sampleRate.HasValue
? $"Records at {_sampleRate} Hz (e.g. for speech or high-fidelity)."
: "Uses the browser's default sample rate (typically 48 kHz).")
| input
| (audioFile.Value != null
? Text.P($"Uploaded: {StringHelper.FormatBytes(audioFile.Value.Length)}").Small()
: null);
}
}

public class AudioInputDisabledState : ViewBase
{
public override object? Build()
Expand Down
10 changes: 9 additions & 1 deletion src/Ivy/Widgets/AudioInput.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@ namespace Ivy;
/// </summary>
public record AudioInput : WidgetBase<AudioInput>
{
public AudioInput(UploadContext upload, string? label = null, string? recordingLabel = null, string mimeType = "audio/webm", int? chunkInterval = null, bool disabled = false)
public AudioInput(UploadContext upload, string? label = null, string? recordingLabel = null, string mimeType = "audio/webm", int? chunkInterval = null, int? sampleRate = null, bool disabled = false)
{
UploadUrl = upload.UploadUrl;
Label = label;
RecordingLabel = recordingLabel;
MimeType = mimeType;
ChunkInterval = chunkInterval;
SampleRate = sampleRate;
Disabled = disabled;
}

Expand All @@ -30,6 +31,8 @@ internal AudioInput() { }

[Prop] public int? ChunkInterval { get; set; }

[Prop] public int? SampleRate { get; set; }

[Prop] public string? UploadUrl { get; set; }
}

Expand Down Expand Up @@ -60,6 +63,11 @@ public static AudioInput ChunkInterval(this AudioInput widget, int? chunkInterva
return widget with { ChunkInterval = chunkInterval };
}

public static AudioInput SampleRate(this AudioInput widget, int? sampleRate)
{
return widget with { SampleRate = sampleRate };
}

public static AudioInput UploadUrl(this AudioInput widget, string? uploadUrl)
{
return widget with { UploadUrl = uploadUrl };
Expand Down
40 changes: 36 additions & 4 deletions src/frontend/src/widgets/inputs/AudioInputWidget.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ interface AudioInputWidgetProps {
width?: string;
uploadUrl: string;
chunkInterval: number;
sampleRate?: number | null;
density?: Densities;
}

Expand All @@ -48,6 +49,7 @@ export const AudioInputWidget: React.FC<AudioInputWidgetProps> = ({
width,
uploadUrl,
chunkInterval = 1000,
sampleRate,
density = Densities.Medium,
}) => {
const normalizedMimeTypes = useMemo(() => {
Expand Down Expand Up @@ -138,6 +140,7 @@ export const AudioInputWidget: React.FC<AudioInputWidgetProps> = ({
if (cancelled) {
return;
}

const mediaRecorderAvailable = typeof MediaRecorder !== 'undefined';
const canProbeTypeSupport =
mediaRecorderAvailable &&
Expand Down Expand Up @@ -174,7 +177,38 @@ export const AudioInputWidget: React.FC<AudioInputWidgetProps> = ({

selectedMimeTypeRef.current = supportedMimeType;

const mediaRecorder = new MediaRecorder(stream, {
const audioContext =
sampleRate != null
? new AudioContext({ sampleRate })
: new AudioContext();
if (audioContext.state === 'suspended') {
await audioContext.resume();
}
if (cancelled) return;
const source = audioContext.createMediaStreamSource(stream);

let streamToRecord: MediaStream;
if (sampleRate != null) {
const destination = audioContext.createMediaStreamDestination();
source.connect(destination);
streamToRecord = destination.stream;
const micRate = stream
.getAudioTracks()[0]
?.getSettings?.()?.sampleRate;
logger.warn(
`AudioInput: requested ${sampleRate} Hz, mic ${micRate ?? '?'} Hz - recording at ${audioContext.sampleRate} Hz (resampled)`
);
} else {
streamToRecord = stream;
const micRate = stream
.getAudioTracks()[0]
?.getSettings?.()?.sampleRate;
logger.warn(
`AudioInput: no sample rate set, recording at ${micRate ?? audioContext.sampleRate} Hz (mic default)`
);
}

const mediaRecorder = new MediaRecorder(streamToRecord, {
mimeType: supportedMimeType,
});

Expand All @@ -187,8 +221,6 @@ export const AudioInputWidget: React.FC<AudioInputWidgetProps> = ({
mediaRecorder.start(chunkInterval);
setRecordingStartedAt(Date.now());

const audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
const analyser = audioContext.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
Expand Down Expand Up @@ -219,7 +251,7 @@ export const AudioInputWidget: React.FC<AudioInputWidgetProps> = ({
onCancel();
setRecordingStoppedAt(Date.now());
};
}, [recording, chunkInterval, uploadChunk, normalizedMimeTypes]);
}, [recording, chunkInterval, sampleRate, uploadChunk, normalizedMimeTypes]);

const volumePercent = recording ? Math.min(volume / 255, 1) * 100 : 0;

Expand Down
Loading