Skip to content

Commit 8a730e7

Browse files
radkostanevradkostanev
committed
chore: add speech to text button integration with MEAI
1 parent e8573d5 commit 8a730e7

File tree

17 files changed

+567
-0
lines changed

17 files changed

+567
-0
lines changed

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration.sln

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.14.36109.1 d17.14
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SpeechToTextIntegration", "SpeechToTextIntegration\SpeechToTextIntegration.csproj", "{3F2BEC52-4F23-42C6-8791-3DC6CA813DB1}"
7+
EndProject
8+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Telerik.Blazor", "..\..\..\..\blazor\Telerik.Blazor\Telerik.Blazor.csproj", "{AF9263B3-0FD2-6644-74FE-84A802165E95}"
9+
EndProject
10+
Global
11+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
12+
Debug|Any CPU = Debug|Any CPU
13+
Release|Any CPU = Release|Any CPU
14+
EndGlobalSection
15+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
16+
{3F2BEC52-4F23-42C6-8791-3DC6CA813DB1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
17+
{3F2BEC52-4F23-42C6-8791-3DC6CA813DB1}.Debug|Any CPU.Build.0 = Debug|Any CPU
18+
{3F2BEC52-4F23-42C6-8791-3DC6CA813DB1}.Release|Any CPU.ActiveCfg = Release|Any CPU
19+
{3F2BEC52-4F23-42C6-8791-3DC6CA813DB1}.Release|Any CPU.Build.0 = Release|Any CPU
20+
{AF9263B3-0FD2-6644-74FE-84A802165E95}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
21+
{AF9263B3-0FD2-6644-74FE-84A802165E95}.Debug|Any CPU.Build.0 = Debug|Any CPU
22+
{AF9263B3-0FD2-6644-74FE-84A802165E95}.Release|Any CPU.ActiveCfg = Release|Any CPU
23+
{AF9263B3-0FD2-6644-74FE-84A802165E95}.Release|Any CPU.Build.0 = Release|Any CPU
24+
EndGlobalSection
25+
GlobalSection(SolutionProperties) = preSolution
26+
HideSolutionNode = FALSE
27+
EndGlobalSection
28+
GlobalSection(ExtensibilityGlobals) = postSolution
29+
SolutionGuid = {1E0CB172-1F2C-4A5B-8DC3-67C1D8A23B53}
30+
EndGlobalSection
31+
EndGlobal

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/App.razor

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
4+
<head>
5+
<meta charset="utf-8" />
6+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7+
<base href="/" />
8+
<link rel="stylesheet" href="bootstrap/bootstrap.min.css" />
9+
<link rel="stylesheet" href="app.css" />
10+
<link rel="stylesheet" href="SpeechToTextIntegration.styles.css" />
11+
<link rel="icon" type="image/png" href="favicon.png" />
12+
<link href="https://unpkg.com/@@progress/kendo-theme-default@@11.0.1/dist/default-main.css" rel="stylesheet" />
13+
<HeadOutlet @rendermode="InteractiveServer" />
14+
</head>
15+
16+
<body>
17+
<Routes @rendermode="InteractiveServer" />
18+
<script src="_framework/blazor.web.js"></script>
19+
</body>
20+
21+
</html>

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/Layout/MainLayout.razor

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
@inherits LayoutComponentBase
2+
3+
<div class="page">
4+
<main>
5+
<article class="content px-4">
6+
@Body
7+
</article>
8+
</main>
9+
</div>
10+
11+
<div id="blazor-error-ui">
12+
An unhandled error has occurred.
13+
<a href="" class="reload">Reload</a>
14+
<a class="dismiss">🗙</a>
15+
</div>

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/Layout/MainLayout.razor.css

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
.page {
2+
position: relative;
3+
display: flex;
4+
flex-direction: column;
5+
}
6+
7+
main {
8+
flex: 1;
9+
}
10+
11+
.sidebar {
12+
background-image: linear-gradient(180deg, rgb(5, 39, 103) 0%, #3a0647 70%);
13+
}
14+
15+
.top-row {
16+
background-color: #f7f7f7;
17+
border-bottom: 1px solid #d6d5d5;
18+
justify-content: flex-end;
19+
height: 3.5rem;
20+
display: flex;
21+
align-items: center;
22+
}
23+
24+
.top-row ::deep a, .top-row ::deep .btn-link {
25+
white-space: nowrap;
26+
margin-left: 1.5rem;
27+
text-decoration: none;
28+
}
29+
30+
.top-row ::deep a:hover, .top-row ::deep .btn-link:hover {
31+
text-decoration: underline;
32+
}
33+
34+
.top-row ::deep a:first-child {
35+
overflow: hidden;
36+
text-overflow: ellipsis;
37+
}
38+
39+
@media (max-width: 640.98px) {
40+
.top-row {
41+
justify-content: space-between;
42+
}
43+
44+
.top-row ::deep a, .top-row ::deep .btn-link {
45+
margin-left: 0;
46+
}
47+
}
48+
49+
@media (min-width: 641px) {
50+
.page {
51+
flex-direction: row;
52+
}
53+
54+
.sidebar {
55+
width: 250px;
56+
height: 100vh;
57+
position: sticky;
58+
top: 0;
59+
}
60+
61+
.top-row {
62+
position: sticky;
63+
top: 0;
64+
z-index: 1;
65+
}
66+
67+
.top-row.auth ::deep a:first-child {
68+
flex: 1;
69+
text-align: right;
70+
width: 0;
71+
}
72+
73+
.top-row, article {
74+
padding-left: 2rem !important;
75+
padding-right: 1.5rem !important;
76+
}
77+
}
78+
79+
#blazor-error-ui {
80+
background: lightyellow;
81+
bottom: 0;
82+
box-shadow: 0 -1px 2px rgba(0, 0, 0, 0.2);
83+
display: none;
84+
left: 0;
85+
padding: 0.6rem 1.25rem 0.7rem 1.25rem;
86+
position: fixed;
87+
width: 100%;
88+
z-index: 1000;
89+
}
90+
91+
#blazor-error-ui .dismiss {
92+
cursor: pointer;
93+
position: absolute;
94+
right: 0.75rem;
95+
top: 0.5rem;
96+
}

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/Pages/Home.razor

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
@page "/"
2+
@using Microsoft.Extensions.AI
3+
@inject IJSRuntime JSRuntime
4+
5+
@inject ISpeechToTextClient SpeechToTextClient
6+
7+
<TelerikTextArea @bind-Value="@TextValue"
8+
Width="300px"
9+
ShowSuffixSeparator="false">
10+
<TextAreaSuffixTemplate>
11+
<span class="k-spacer"></span>
12+
<TelerikSpeechToTextButton OnStart="@OnStartHandler"
13+
OnEnd="@OnEndHandler"
14+
FillMode="@ThemeConstants.Button.FillMode.Flat"
15+
IntegrationMode="@SpeechToTextButtonIntegrationMode.None">
16+
</TelerikSpeechToTextButton>
17+
</TextAreaSuffixTemplate>
18+
</TelerikTextArea>
19+
20+
21+
22+
@code {
23+
private string TextValue { get; set; } = string.Empty;
24+
private DotNetObjectReference<Home>? dotNetObjectReference;
25+
26+
private async void OnStartHandler()
27+
{
28+
await JSRuntime.InvokeVoidAsync("speechRecognitionStarted");
29+
}
30+
31+
private async void OnEndHandler()
32+
{
33+
await JSRuntime.InvokeVoidAsync("speechRecognitionEnded");
34+
}
35+
36+
protected override async Task OnAfterRenderAsync(bool firstRender)
37+
{
38+
if (firstRender)
39+
{
40+
await JSRuntime.InvokeVoidAsync("initializeSpeechToTextButton");
41+
42+
dotNetObjectReference = DotNetObjectReference.Create(this);
43+
44+
await JSRuntime.InvokeVoidAsync("setDotNetObjectReference", dotNetObjectReference);
45+
}
46+
47+
await base.OnAfterRenderAsync(firstRender);
48+
}
49+
50+
[JSInvokable("OnRecordedAudio")]
51+
public async Task OnRecordedAudio(byte[] audioBytes)
52+
{
53+
if (audioBytes == null || audioBytes.Length == 0)
54+
{
55+
return;
56+
}
57+
58+
using var stream = new MemoryStream(audioBytes);
59+
60+
try
61+
{
62+
await GetSpeechToTextResponse(stream);
63+
}
64+
catch (Exception e)
65+
{
66+
Console.WriteLine(e.Message);
67+
return;
68+
}
69+
}
70+
71+
private async Task GetSpeechToTextResponse(MemoryStream stream)
72+
{
73+
var response = await SpeechToTextClient.GetTextAsync(stream);
74+
TextValue = response.Text;
75+
StateHasChanged();
76+
}
77+
}
78+
79+
<script>
80+
// Function to initialize the speechToTextButton object
81+
window.initializeSpeechToTextButton = function() {
82+
console.log("Initializing speechToTextButton object...");
83+
84+
// Create a dedicated object for speech-to-text functionality
85+
window.speechToTextButton = {
86+
// Properties
87+
mediaRecorder: null,
88+
recordingAborted: false,
89+
audioChunks: [],
90+
stream: null,
91+
92+
// Methods
93+
bindMediaRecorderEvents() {
94+
console.log("Binding media recorder events...");
95+
this.mediaRecorder.onstart = () => this.onStart();
96+
this.mediaRecorder.ondataavailable = (e) => this.audioChunks.push(e.data);
97+
this.mediaRecorder.onstop = async () => {
98+
if (this.mediaRecorder) {
99+
if (!this.recordingAborted) {
100+
const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
101+
const arrayBuffer = await audioBlob.arrayBuffer();
102+
const uint8Array = new Uint8Array(arrayBuffer);
103+
// Call back to Blazor with the recorded audio data
104+
try {
105+
window.dotNetObjectReference.invokeMethodAsync("OnRecordedAudio", uint8Array);
106+
console.log("Successfully called OnRecordedAudio via component reference");
107+
} catch (error) {
108+
console.error("Error calling OnRecordedAudio:", error);
109+
}
110+
}
111+
this.audioChunks = [];
112+
this.unbindMediaRecorderEvents();
113+
this.onEnd();
114+
}
115+
};
116+
},
117+
118+
unbindMediaRecorderEvents() {
119+
console.log("Unbinding media recorder events...");
120+
if (this.stream) {
121+
this.stream.getTracks().forEach(track => track.stop());
122+
this.stream = null;
123+
}
124+
if (this.mediaRecorder) {
125+
this.mediaRecorder.onstart = null;
126+
this.mediaRecorder.ondataavailable = null;
127+
this.mediaRecorder.onstop = null;
128+
this.mediaRecorder.onerror = null;
129+
if (this.mediaRecorder.stream) {
130+
this.mediaRecorder.stream.getTracks().forEach(track => track.stop());
131+
}
132+
this.mediaRecorder = null;
133+
}
134+
},
135+
136+
async startMediaRecorder() {
137+
console.log("Starting media recorder...");
138+
this.recordingAborted = false;
139+
this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
140+
this.mediaRecorder = new MediaRecorder(this.stream);
141+
this.bindMediaRecorderEvents();
142+
this.mediaRecorder.start();
143+
},
144+
145+
async stopMediaRecorder() {
146+
console.log("Stopping media recorder...");
147+
if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') {
148+
this.mediaRecorder.stop();
149+
}
150+
},
151+
152+
// Event callbacks
153+
onStart() {
154+
// add any additional logic here if necessary
155+
console.log("Media recorder started");
156+
},
157+
158+
onEnd() {
159+
// add any additional logic here if necessary
160+
console.log("Media recorder ended");
161+
},
162+
163+
// Public API methods
164+
async speechRecognitionStarted() {
165+
console.log("Speech recognition started - called from Blazor");
166+
await this.startMediaRecorder();
167+
},
168+
169+
async speechRecognitionEnded() {
170+
console.log("Speech recognition ended - called from Blazor");
171+
await this.stopMediaRecorder();
172+
},
173+
};
174+
175+
// Expose the API methods to window for Blazor interop
176+
window.speechRecognitionStarted = () => window.speechToTextButton.speechRecognitionStarted();
177+
window.speechRecognitionEnded = () => window.speechToTextButton.speechRecognitionEnded();
178+
window.setDotNetObjectReference = (value) => window.dotNetObjectReference = value;
179+
180+
console.log("speechToTextButton object initialized successfully");
181+
};
182+
183+
</script>

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/Routes.razor

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<Router AppAssembly="typeof(Program).Assembly">
2+
<Found Context="routeData">
3+
<RouteView RouteData="routeData" DefaultLayout="typeof(Layout.MainLayout)" />
4+
<FocusOnNavigate RouteData="routeData" Selector="h1" />
5+
</Found>
6+
</Router>

‎common/microsoft-extensions-ai-integration/SpeechToTextIntegration/SpeechToTextIntegration/Components/_Imports.razor

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
@using System.Net.Http
2+
@using System.Net.Http.Json
3+
@using Microsoft.AspNetCore.Components.Forms
4+
@using Microsoft.AspNetCore.Components.Routing
5+
@using Microsoft.AspNetCore.Components.Web
6+
@using static Microsoft.AspNetCore.Components.Web.RenderMode
7+
@using Microsoft.AspNetCore.Components.Web.Virtualization
8+
@using Microsoft.JSInterop
9+
@using SpeechToTextIntegration
10+
@using SpeechToTextIntegration.Components
11+
12+
@using Telerik.Blazor
13+
@using Telerik.Blazor.Components

0 commit comments

Comments
 (0)