chore: wip

Author: radkostanev

common/microsoft-extensions-ai-integration/SpeechToTextIntegration/README.md

@@ -0,0 +1,16 @@
+# SpeechToTextIntegration Demo
+
+This project demonstrates the integration of the Telerik UI for Blazor `SpeechToTextButton` component with a transcription model, such as OpenAI's `whisper-1`. It provides a simple Blazor UI for recording audio and transcribing speech to text, showcasing how to connect the UI component to a backend speech-to-text service.
+
+## Main Purpose
+- **Showcase**: Illustrates how to use the Telerik `SpeechToTextButton` in a Blazor application.
+- **Integration**: Demonstrates sending recorded audio to a transcription model (e.g., OpenAI Whisper) and displaying the transcribed text in the UI.
+- **Extensibility**: Serves as a starting point for integrating other speech-to-text models or services.
+
+## Configuration Notes
+- **Model Registration**: The setup for registering a transcription model (such as OpenAI Whisper or others) may vary. Refer to the specific model's documentation for registration and authentication steps.
+- **Audio Recording**: The requirements for the recorded audio (file size, type, encoding, etc.) depend on the chosen transcription model. Ensure that the audio format produced by the UI matches the model's expected input.
+- **Customization**: You may need to adjust the audio recording logic or backend integration to support different models or to optimize for accuracy and performance.
+
+---
+For more details, see the source code and comments in the `Home.razor` component.

common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/App.razor

@@ -10,6 +10,7 @@
     <link rel="stylesheet" href="SpeechToTextIntegration.styles.css" />
     <link rel="icon" type="image/png" href="favicon.png" />
     <link href="https://unpkg.com/@@progress/kendo-theme-default@@11.0.1/dist/default-main.css" rel="stylesheet" />
+    <script src="_content/Telerik.UI.for.Blazor/js/telerik-blazor.js" defer></script>
     <HeadOutlet @rendermode="InteractiveServer" />
 </head>
 

common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/Pages/Home.razor

@@ -1,7 +1,8 @@
 @page "/"
+
 @using Microsoft.Extensions.AI
-@inject IJSRuntime JSRuntime
 
+@inject IJSRuntime JSRuntime
 @inject ISpeechToTextClient SpeechToTextClient
 
 <TelerikTextArea @bind-Value="@TextValue"
@@ -17,18 +18,16 @@
     </TextAreaSuffixTemplate>
 </TelerikTextArea>
 
-
-
 @code {
     private string TextValue { get; set; } = string.Empty;
     private DotNetObjectReference<Home>? dotNetObjectReference;
 
-    private async void OnStartHandler()
+    private async Task OnStartHandler()
     {
         await JSRuntime.InvokeVoidAsync("speechRecognitionStarted");
     }
 
-    private async void OnEndHandler()
+    private async Task OnEndHandler()
     {
         await JSRuntime.InvokeVoidAsync("speechRecognitionEnded");
     }
@@ -37,11 +36,18 @@
     {
         if (firstRender)
         {
-            await JSRuntime.InvokeVoidAsync("initializeSpeechToTextButton");
-
-            dotNetObjectReference = DotNetObjectReference.Create(this);
-
-            await JSRuntime.InvokeVoidAsync("setDotNetObjectReference", dotNetObjectReference);
+            try
+            {
+                await JSRuntime.InvokeVoidAsync("initializeSpeechToTextButton");
+
+                dotNetObjectReference = DotNetObjectReference.Create(this);
+
+                await JSRuntime.InvokeVoidAsync("setDotNetObjectReference", dotNetObjectReference);
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine($"JSInterop failed: {ex.Message}");
+            }
         }
 
         await base.OnAfterRenderAsync(firstRender);
@@ -55,10 +61,10 @@
             return;
         }
 
-        using var stream = new MemoryStream(audioBytes);
-
         try
         {
+            using var stream = new MemoryStream(audioBytes);
+
             await GetSpeechToTextResponse(stream);
         }
         catch (Exception e)
@@ -96,17 +102,18 @@
                 this.mediaRecorder.ondataavailable = (e) => this.audioChunks.push(e.data);
                 this.mediaRecorder.onstop = async () => {
                     if (this.mediaRecorder) {
-                        if (!this.recordingAborted) {
-                            const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
-                            const arrayBuffer = await audioBlob.arrayBuffer();
-                            const uint8Array = new Uint8Array(arrayBuffer);
-                            // Call back to Blazor with the recorded audio data
-                            try {
-                                window.dotNetObjectReference.invokeMethodAsync("OnRecordedAudio", uint8Array);
-                                console.log("Successfully called OnRecordedAudio via component reference");
-                            } catch (error) {
-                                console.error("Error calling OnRecordedAudio:", error);
+                        const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
+                        const arrayBuffer = await audioBlob.arrayBuffer();
+                        const uint8Array = new Uint8Array(arrayBuffer);
+                        // Call back to Blazor with the recorded audio data
+                        try {
+                            if (window.dotNetObjectReference) {
+                                await window.dotNetObjectReference.invokeMethodAsync("OnRecordedAudio", uint8Array);
+                            } else {
+                                console.warn("dotNetObjectReference is not set.");
                             }
+                        } catch (error) {
+                            console.error("Error calling OnRecordedAudio:", error);
                         }
                         this.audioChunks = [];
                         this.unbindMediaRecorderEvents();
@@ -152,12 +159,12 @@
             // Event callbacks
             onStart() {
                 // add any additional logic here if necessary
-                console.log("Media recorder started"); 
+                console.log("Media recorder started");
             },
 
             onEnd() {
                 // add any additional logic here if necessary
-                console.log("Media recorder ended"); 
+                console.log("Media recorder ended");
             },
 
             // Public API methods

common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Program.cs

@@ -5,16 +5,17 @@
 
 // Add services to the container.
 builder.Services.AddRazorComponents()
-    .AddInteractiveServerComponents();
+    .AddInteractiveServerComponents()
+    .AddHubOptions(o => o.MaximumReceiveMessageSize = 4 * 1024 * 1024);
 
 builder.Services.AddTelerikBlazor();
 
 #region AI Service Registration Start
 
-var modelName = Environment.GetEnvironmentVariable("OPEN_AI_REPL_ASSISTANT_MODEL_NAME");
-var key = Environment.GetEnvironmentVariable("OPEN_AI_REPL_ASSISTANT_MODEL_API_KEY");
+// Get the appropriate environment variables for your model's service.
+var key = Environment.GetEnvironmentVariable("YOUR_TRANSCRIPTION_MODEL_API_KEY");
 
-// Open AI whisper-1 model registration
+// Open AI whisper-1 model registration. Refer to your preferred model documentation for more details.
 builder.Services.AddDistributedMemoryCache();
 builder.Services.AddSpeechToTextClient(services =>
     new OpenAI.Audio.AudioClient("whisper-1", key).AsISpeechToTextClient());
@@ -32,6 +33,7 @@
 }
 
 app.UseHttpsRedirection();
+app.UseDeveloperExceptionPage();
 
 app.UseStaticFiles();
 app.UseAntiforgery();

common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/appsettings.json

@@ -1,4 +1,5 @@
 {
+  "DetailedErrors": true,
   "Logging": {
     "LogLevel": {
       "Default": "Information",