Openai demo trigger conv (#1288)

* wip: trigger conversation with dc call in openai demo.

* Now triggering the openai demo to automatically start talking.

Author: sipsorcery

examples/WebRTCExamples/WebRTCOpenAI/Program.cs

@@ -38,6 +38,7 @@
 // History:
 // 19 Dec 2024	Aaron Clauson	Created, Dublin, Ireland.
 // 28 Dec 2024  Aaron Clauson   Switched to functional approach for The Craic.
+// 17 Jan 2025  Aaron Clauson   Added create resposne data channel message to trigger conversation start.
 //
 // License: 
 // BSD 3-Clause "New" or "Revised" License, see included LICENSE.md file.
@@ -67,17 +68,30 @@ record Problem(string detail);
 
     record PcContext(
        RTCPeerConnection Pc,
+       SemaphoreSlim PcConnectedSemaphore,
        string EphemeralKey = "",
        string OfferSdp = "",
        string AnswerSdp = ""
     );
 
+    enum VoicesEnum
+    {
+        alloy,
+        ash,
+        ballad,
+        coral,
+        echo,
+        sage,
+        shimmer,
+        verse
+    }
+
     class Program
     {
         private const string OPENAI_REALTIME_SESSIONS_URL = "https://api.openai.com/v1/realtime/sessions";
         private const string OPENAI_REALTIME_BASE_URL = "https://api.openai.com/v1/realtime";
         private const string OPENAI_MODEL = "gpt-4o-realtime-preview-2024-12-17";
-        private const string OPENAI_VERSE = "shimmer"; // Supported values are: 'alloy', 'ash', 'ballad', 'coral', 'echo', 'sage', 'shimmer', and 'verse'.
+        private const VoicesEnum OPENAI_VERSE = VoicesEnum.shimmer;
         private const string OPENAI_DATACHANNEL_NAME = "oai-events";
 
         private static Microsoft.Extensions.Logging.ILogger logger = NullLogger.Instance;
@@ -101,26 +115,27 @@ static async Task Main(string[] args)
                 .BindAsync(_ =>
                 {
                     logger.LogInformation("STEP 1: Get ephemeral key from OpenAI.");
-                    return CreateEphemeralKeyAsync(OPENAI_REALTIME_SESSIONS_URL, args[0], OPENAI_MODEL, OPENAI_VERSE);
+                    return CreateEphemeralKeyAsync(OPENAI_REALTIME_SESSIONS_URL, args[0], OPENAI_MODEL, OPENAI_VERSE.ToString());
                 })
                 .BindAsync(async ephemeralKey =>
                 {
                     logger.LogDebug("STEP 2: Create WebRTC PeerConnection & get local SDP offer.");
 
-                    var pc = await CreatePeerConnection();
+                    var onConnectedSemaphore = new SemaphoreSlim(0, 1);
+                    var pc = await CreatePeerConnection(onConnectedSemaphore);
                     var offer = pc.createOffer();
                     await pc.setLocalDescription(offer);
 
                     logger.LogDebug("SDP offer:");
                     logger.LogDebug(offer.sdp);
 
                     return Prelude.Right<Problem, PcContext>(
-                        new PcContext(pc, ephemeralKey, offer.sdp, string.Empty)
+                        new PcContext(pc, onConnectedSemaphore, ephemeralKey, offer.sdp, string.Empty)
                     );
                 })
                 .BindAsync(async ctx =>
                 {
-                    logger.LogInformation("STEP 3: Send SDP offer to OpenAI REST server & get SDP answer."); 
+                    logger.LogInformation("STEP 3: Send SDP offer to OpenAI REST server & get SDP answer.");
 
                     var answerEither = await GetOpenAIAnswerSdpAsync(ctx.EphemeralKey, ctx.OfferSdp);
                     return answerEither.Map(answer => ctx with { AnswerSdp = answer });
@@ -141,9 +156,21 @@ static async Task Main(string[] args)
                         Prelude.Right<Problem, PcContext>(ctx) :
                         Prelude.Left<Problem, PcContext>(new Problem("Failed to set remote SDP."));
                 })
+                .MapAsync(async ctx =>
+                {
+                    logger.LogInformation("STEP 5: Wait for data channel to connect and then trigger conversation.");
+
+                    await ctx.PcConnectedSemaphore.WaitAsync();
+
+                    // NOTE: If you want to trigger the convesation by using the audio from your microphone comment
+                    // out this line.
+                    SendResponseCreate(ctx.Pc.DataChannels.First(), VoicesEnum.alloy, "Introduce urself.");
+
+                    return ctx;
+                })
                 .BindAsync(ctx =>
                 {
-                    logger.LogInformation("STEP 5: Wait for ctrl-c to indicate user exit.");
+                    logger.LogInformation("STEP 6: Wait for ctrl-c to indicate user exit.");
 
                     ManualResetEvent exitMre = new(false);
                     Console.CancelKeyPress += (_, e) =>
@@ -162,9 +189,35 @@ static async Task Main(string[] args)
                 Left: prob => Console.WriteLine($"There was a problem setting up the connection. {prob.detail}"),
                 Right: _ => Console.WriteLine("The call was successful.")
             );
-       }
+        }
+
+        /// <summary>
+        /// Sends a response create message to the OpenAI data channel to trigger the conversation.
+        /// </summary>
+        private static void SendResponseCreate(RTCDataChannel dc, VoicesEnum voice, string message)
+        {
+            var responseCreate = new OpenAIResponseCreate
+            {
+                EventID = Guid.NewGuid().ToString(),
+                Response = new OpenAIResponseCreateResponse
+                {
+                    Instructions = message,
+                    Voice = voice.ToString()
+                }
+            };
+
+            logger.LogInformation($"Sending initial response create to first call data channel {dc.label}.");
+            logger.LogDebug(responseCreate.ToJson());
+
+            dc.send(responseCreate.ToJson());
+        }
 
-        private static async Task<RTCPeerConnection> CreatePeerConnection()
+        /// <summary>
+        /// Method to create the local peer connection instance and data channel.
+        /// </summary>
+        /// <param name="onConnectedSemaphore">A semaphore that will get set when the data channel on the peer connection is opened. Since the data channel
+        /// can only be opened once the peer connection is open this indicates both are ready for use.</param>
+        private static async Task<RTCPeerConnection> CreatePeerConnection(SemaphoreSlim onConnectedSemaphore)
         {
             var pcConfig = new RTCConfiguration
             {
@@ -224,6 +277,7 @@ private static async Task<RTCPeerConnection> CreatePeerConnection()
             dataChannel.onopen += () =>
             {
                 logger.LogDebug("OpenAI data channel opened.");
+                onConnectedSemaphore.Release();
             };
 
             dataChannel.onclose += () => logger.LogDebug($"OpenAI data channel {dataChannel.label} closed.");
@@ -233,6 +287,9 @@ private static async Task<RTCPeerConnection> CreatePeerConnection()
             return peerConnection;
         }
 
+        /// <summary>
+        /// Event handler for WebRTC data channel messages.
+        /// </summary>
         private static void OnDataChannelMessage(RTCDataChannel dc, DataChannelPayloadProtocols protocol, byte[] data)
         {
             //logger.LogInformation($"Data channel {dc.label}, protocol {protocol} message length {data.Length}.");
@@ -265,6 +322,10 @@ private static void OnDataChannelMessage(RTCDataChannel dc, DataChannelPayloadPr
             }
         }
 
+        /// <summary>
+        /// Completes the steps required to get an ephemeral key from the OpenAI REST server. The ephemeral key is needed
+        /// to send an SDP offer, and get the SDP answer.
+        /// </summary>
         private static async Task<Either<Problem, string>> CreateEphemeralKeyAsync(string sessionsUrl, string openAIToken, string model, string voice)
             => (await SendHttpPostAsync(
                 sessionsUrl,
@@ -284,13 +345,23 @@ private static async Task<Either<Problem, string>> CreateEphemeralKeyAsync(strin
                 Prelude.Left<Problem, string>(new Problem("Failed to get ephemeral secret."))
             );
 
+        /// <summary>
+        /// Attempts to get the SDP answer from the OpenAI REST server. This is the way OpenAI does the signalling. The
+        /// ICE candidates will be returned in the SDP answer and are publicly accessible IP's.
+        /// </summary>
+        /// <remarks>
+        /// See https://platform.openai.com/docs/guides/realtime-webrtc#creating-an-ephemeral-token.
+        /// </remarks>
         private static Task<Either<Problem, string>> GetOpenAIAnswerSdpAsync(string ephemeralKey, string offerSdp)
             => SendHttpPostAsync(
                 $"{OPENAI_REALTIME_BASE_URL}?model={OPENAI_MODEL}",
                 ephemeralKey,
                 offerSdp,
                 "application/sdp");
 
+        /// <summary>
+        /// Helper method to send an HTTP psot request with the required headers.
+        /// </summary>
         private static async Task<Either<Problem, string>> SendHttpPostAsync(
             string url,
             string token,

examples/WebRTCExamples/WebRTCOpenAI/RealtimeModels/OpenAIResponseCreate.cs

@@ -0,0 +1,35 @@
+// https://platform.openai.com/docs/api-reference/realtime-client-events/response/create
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace demo;
+
+public class OpenAIResponseCreate : OpenAIServerEventBase
+{
+    [JsonPropertyName("type")]
+    public override string Type => "response.create";
+
+    [JsonPropertyName("response")]
+    public required OpenAIResponseCreateResponse Response { get; set; }
+
+    public override string ToJson()
+    {
+        return JsonSerializer.Serialize(this, JsonOptions.Default);
+    }
+}
+
+public class OpenAIResponseCreateResponse
+{
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = new[] { "audio", "text" };
+
+    [JsonPropertyName("instructions")]
+    public string? Instructions { get; set; }
+
+    [JsonPropertyName("voice")]
+    public string? Voice { get; set; }
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFrmat { get; set; } = "pcm16";
+}

examples/WebRTCExamples/WebRTCOpenAI/RealtimeModels/OpenAIServerEventBase.cs

@@ -8,7 +8,8 @@ public class OpenAIServerEventBase
     [JsonPropertyName("event_id")]
     public string? EventID { get; set; }
 
-    public string? Type { get; set; }
+    [JsonPropertyName("type")]
+    public virtual string? Type { get; set; }
 
     public virtual string ToJson()
     {

examples/WebRTCExamples/WebRTCOpenAI/WebRTCOpenAI.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net8.0-windows10.0.17763.0</TargetFramework>
-    <LangVersion>10.0</LangVersion>
+    <LangVersion>12.0</LangVersion>
     <Nullable>enable</Nullable>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>

examples/WebRTCExamples/WebRTCOpenAIAliceAndBob/Program.cs

@@ -192,20 +192,10 @@ from bobCtx in bobCallTask.Result
                 // Trigger the conversation by getting Alice to say something witty.
                 var aliceDataChannel = aliceConncectedCtx.Pc.DataChannels.Where(x => x.label == OPENAI_DATACHANNEL_NAME).Single();
 
-                var responseCreate = new OpenAIResponseCreate
+                if (aliceDataChannel != null)
                 {
-                    EventID = Guid.NewGuid().ToString(),
-                    Response = new OpenAIResponseCreateResponse
-                    {
-                        Instructions = "Only talk in cheesy puns. Keep it short once you'vegot you pun in. To start the conversation please repeat repeat this phrase in your corniest accent: 'You're a few tinnies short of a six-pack.'",
-                        Voice = VoicesEnum.shimmer.ToString()
-                    }
-                };
-
-                logger.LogInformation($"Sending initial response create to first call data channel {aliceDataChannel?.label}.");
-                logger.LogDebug(responseCreate.ToJson());
-
-                aliceDataChannel?.send(responseCreate.ToJson());
+                    SendResponseCreate(aliceDataChannel, VoicesEnum.shimmer, "Only talk in cheesy puns. Keep it short once you'vegot you pun in. To start the conversation please repeat repeat this phrase in your corniest accent: 'You're a few tinnies short of a six-pack.'");
+                }
 
                 logger.LogInformation($"ctrl-c to exit..");
 
@@ -229,6 +219,27 @@ from bobCtx in bobCallTask.Result
             _audioScopeForm?.Invoke(() => _audioScopeForm.Close());
         }
 
+        /// <summary>
+        /// Sends a response create message to the OpenAI data channel to trigger the conversation.
+        /// </summary>
+        private static void SendResponseCreate(RTCDataChannel dc, VoicesEnum voice, string message)
+        {
+            var responseCreate = new OpenAIResponseCreate
+            {
+                EventID = Guid.NewGuid().ToString(),
+                Response = new OpenAIResponseCreateResponse
+                {
+                    Instructions = message,
+                    Voice = voice.ToString()
+                }
+            };
+
+            logger.LogInformation($"Sending initial response create to first call data channel {dc.label}.");
+            logger.LogDebug(responseCreate.ToJson());
+
+            dc.send(responseCreate.ToJson());
+        }
+
         /// <summary>
         /// Initiaites the creation and media session wiring for a local peer connection.
         /// </summary>