OpenAI STT relay in our proxy with SSE support

.github/workflows/cli_desktop_ci.yml

@@ -17,6 +17,8 @@ jobs:
     defaults:
       run:
         shell: bash
+    env:
+      LIBONNXRUNTIME_NO_PKG_CONFIG: "1"
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_cli_deps
@@ -25,5 +27,5 @@ jobs:
           components: clippy
       - uses: Swatinem/rust-cache@v2
       - run: cargo check -p cli --features desktop
-      - run: cargo clippy -p cli --features desktop -- -D warnings
+      - run: cargo clippy -p cli --features desktop
       - run: cargo test -p cli --features desktop

apps/cli/src/stt/config.rs

@@ -78,6 +78,7 @@ impl ResolvedSttConfig {
             num_speakers: None,
             min_speakers: None,
             max_speakers: None,
+            known_speaker_references: vec![],
         }
     }
 }

crates/cactus/src/llm/complete.rs

@@ -152,7 +152,7 @@ impl Model {
         };
 
         let (rc, buf) = self.call_complete(
-            &guard,
+            guard,
             &request.messages_c,
             &request.options_c,
             Some(token_trampoline::<F>),

crates/listener-core/src/actors/source/pipeline.rs

@@ -14,6 +14,7 @@
 use hypr_vad_masking::VadMask;
 
 use super::{ListenerRouting, SourceFrame};
+use hypr_audio::CaptureFrame;
 
 const AUDIO_AMPLITUDE_THROTTLE: Duration = Duration::from_millis(100);
 const MAX_BUFFER_CHUNKS: usize = 150;

crates/listener2-core/src/batch/mod.rs

@@ -55,6 +55,8 @@ pub struct BatchParams {
     pub min_speakers: Option<u32>,
     #[serde(default)]
     pub max_speakers: Option<u32>,
+    #[serde(default)]
+    pub known_speaker_references: Vec<owhisper_interface::KnownSpeakerReference>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -235,6 +237,7 @@ fn build_listen_params(
         num_speakers: params.num_speakers,
         min_speakers: params.min_speakers,
         max_speakers: params.max_speakers,
+        known_speaker_references: params.known_speaker_references.clone(),
         custom_query: None,
     }
 }
@@ -326,6 +329,7 @@ mod tests {
             num_speakers: None,
             min_speakers: None,
             max_speakers: None,
+            known_speaker_references: vec![],
         }
     }
 
@@ -353,6 +357,20 @@ mod tests {
         assert!(listen_params.custom_query.is_none());
     }
 
+    #[test]
+    fn build_listen_params_preserves_known_speaker_references() {
+        let mut params = batch_params(BatchProvider::OpenAI, "https://api.openai.com/v1");
+        params.known_speaker_references = vec![owhisper_interface::KnownSpeakerReference {
+            name: "agent".to_string(),
+            audio_data_url: "data:audio/wav;base64,AAA=".to_string(),
+        }];
+
+        let listen_params = build_listen_params(&params, 1, 16_000);
+
+        assert_eq!(listen_params.known_speaker_references.len(), 1);
+        assert_eq!(listen_params.known_speaker_references[0].name, "agent");
+    }
+
     #[test]
     fn am_routes_pyannote_to_direct_batch() {
         let params = batch_params(BatchProvider::Am, "https://api.pyannote.ai");

crates/openai-transcription/src/batch/response.rs

@@ -8,6 +8,9 @@ pub enum ParsedTranscriptionStreamEvent {
         partial_text: String,
         logprobs: Vec<TranscriptionLogprob>,
     },
+    TextSegment {
+        segment: DiarizedTranscriptionSegment,
+    },
     TextDone {
         text: String,
         logprobs: Vec<TranscriptionLogprob>,
@@ -65,6 +68,22 @@ impl TranscriptionStreamEventParser {
                     })
                 }
             }
+            TranscriptionStreamEvent::TextSegment {
+                id,
+                end,
+                speaker,
+                start,
+                text,
+            } => Some(ParsedTranscriptionStreamEvent::TextSegment {
+                segment: DiarizedTranscriptionSegment {
+                    id,
+                    end,
+                    speaker,
+                    start,
+                    text,
+                    segment_type: TranscriptionDiarizedSegmentType::TranscriptTextSegment,
+                },
+            }),
             TranscriptionStreamEvent::TextDone {
                 text,
                 logprobs,
@@ -266,6 +285,14 @@ pub enum TranscriptionStreamEvent {
         #[serde(default)]
         logprobs: Vec<TranscriptionLogprob>,
     },
+    #[serde(rename = "transcript.text.segment")]
+    TextSegment {
+        id: String,
+        end: f64,
+        speaker: String,
+        start: f64,
+        text: String,
+    },
     #[serde(rename = "transcript.text.done")]
     TextDone {
         text: String,
@@ -366,6 +393,17 @@ mod tests {
             }"#,
         )
         .expect("parse delta");
+        let segment: TranscriptionStreamEvent = serde_json::from_str(
+            r#"{
+                "type": "transcript.text.segment",
+                "id": "seg_001",
+                "start": 0.0,
+                "end": 1.5,
+                "text": "hello there",
+                "speaker": "agent"
+            }"#,
+        )
+        .expect("parse segment");
         let done: TranscriptionStreamEvent = serde_json::from_str(
             r#"{
                 "type": "transcript.text.done",
@@ -382,6 +420,10 @@ mod tests {
         .expect("parse done");
 
         assert!(matches!(delta, TranscriptionStreamEvent::TextDelta { .. }));
+        assert!(matches!(
+            segment,
+            TranscriptionStreamEvent::TextSegment { .. }
+        ));
         assert!(matches!(done, TranscriptionStreamEvent::TextDone { .. }));
     }
 
@@ -410,4 +452,25 @@ mod tests {
             ParsedTranscriptionStreamEvent::TextDone { text, .. } if text == "hello"
         ));
     }
+
+    #[test]
+    fn parser_preserves_diarized_segment_events() {
+        let mut parser = TranscriptionStreamEventParser::new();
+
+        let segment = parser
+            .parse_sse_block(
+                r#"data: {"type":"transcript.text.segment","id":"seg_001","start":0.0,"end":1.5,"text":"hello there","speaker":"agent"}"#,
+            )
+            .expect("parse segment")
+            .expect("expected segment");
+
+        assert!(matches!(
+            segment,
+            ParsedTranscriptionStreamEvent::TextSegment { segment }
+                if segment.id == "seg_001"
+                    && segment.speaker == "agent"
+                    && segment.text == "hello there"
+        ));
+        assert_eq!(parser.partial_text(), "");
+    }
 }