diff --git a/.github/workflows/release-template.yml b/.github/workflows/release-template.yml index 07fdf90..e583fa8 100644 --- a/.github/workflows/release-template.yml +++ b/.github/workflows/release-template.yml @@ -1,12 +1,10 @@ name: Release Template on: - workflow_dispatch: - inputs: - reason: - description: 'the reason for triggering this workflow' - required: false - default: 'manually publish the ecr images and templates' + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] jobs: release_template: runs-on: ubuntu-latest diff --git a/react-native/ios/SwiftChat/Info.plist b/react-native/ios/SwiftChat/Info.plist index 9f8fd9f..7eecde4 100644 --- a/react-native/ios/SwiftChat/Info.plist +++ b/react-native/ios/SwiftChat/Info.plist @@ -37,6 +37,11 @@ Support record videos, voice chat, and summarize content NSPhotoLibraryUsageDescription Support choose pictures and summarize it + UIBackgroundModes + + audio + fetch + UILaunchStoryboardName LaunchScreen UIRequiredDeviceCapabilities diff --git a/react-native/src/api/bedrock-api.ts b/react-native/src/api/bedrock-api.ts index c8afae5..add55fb 100644 --- a/react-native/src/api/bedrock-api.ts +++ b/react-native/src/api/bedrock-api.ts @@ -299,12 +299,12 @@ export const requestAllModels = async (): Promise => { return { imageModel: [], textModel: [] }; } const allModel = await response.json(); - allModel.imageModel.map((item: Model) => ({ + allModel.imageModel = allModel.imageModel.map((item: Model) => ({ modelId: item.modelId, modelName: item.modelName, modelTag: ModelTag.Bedrock, })); - allModel.textModel.map((item: Model) => ({ + allModel.textModel = allModel.textModel.map((item: Model) => ({ modelId: item.modelId, modelName: item.modelName, modelTag: ModelTag.Bedrock, diff --git a/react-native/src/chat/ChatScreen.tsx b/react-native/src/chat/ChatScreen.tsx index 0244f37..7568089 100644 --- a/react-native/src/chat/ChatScreen.tsx +++ b/react-native/src/chat/ChatScreen.tsx @@ -14,7 +14,9 @@ import { TextInput, } from 'react-native'; import { voiceChatService } from './service/VoiceChatService'; -import AudioWaveformComponent from './component/AudioWaveformComponent'; +import AudioWaveformComponent, { + AudioWaveformRef, +} from './component/AudioWaveformComponent'; import { Colors } from 'react-native/Libraries/NewAppScreen'; import { invokeBedrockWithCallBack as invokeBedrockWithCallBack, @@ -108,7 +110,6 @@ function ChatScreen(): React.JSX.Element { const [systemPrompt, setSystemPrompt] = useState( isNovaSonic ? getCurrentVoiceSystemPrompt : getCurrentSystemPrompt ); - const [audioVolume, setAudioVolume] = useState(1); // Audio volume level (1-10) const [showSystemPrompt, setShowSystemPrompt] = useState(true); const [screenDimensions, setScreenDimensions] = useState( Dimensions.get('window') @@ -132,31 +133,33 @@ function ChatScreen(): React.JSX.Element { const usageRef = useRef(usage); const systemPromptRef = useRef(systemPrompt); const drawerTypeRef = useRef(drawerType); - const inputAudioLevelRef = useRef(1); - const outputAudioLevelRef = useRef(1); const isVoiceLoading = useRef(false); const contentHeightRef = useRef(0); const containerHeightRef = useRef(0); const isNovaSonicRef = useRef(isNovaSonic); const [isShowVoiceLoading, setIsShowVoiceLoading] = useState(false); + const audioWaveformRef = useRef(null); + + const endVoiceConversationRef = useRef<(() => Promise) | null>(null); - // End voice conversation and reset audio levels const endVoiceConversation = useCallback(async () => { + audioWaveformRef.current?.resetAudioLevels(); if (isVoiceLoading.current) { return Promise.resolve(false); } isVoiceLoading.current = true; setIsShowVoiceLoading(true); await voiceChatService.endConversation(); - setAudioVolume(1); - inputAudioLevelRef.current = 1; - outputAudioLevelRef.current = 1; setChatStatus(ChatStatus.Init); isVoiceLoading.current = false; setIsShowVoiceLoading(false); return true; }, []); + useEffect(() => { + endVoiceConversationRef.current = endVoiceConversation; + }, [endVoiceConversation]); + // update refs value with state useEffect(() => { messagesRef.current = messages; @@ -187,23 +190,10 @@ function ChatScreen(): React.JSX.Element { message => { if (getTextModel().modelId.includes('nova-sonic')) { handleVoiceChatTranscript('ASSISTANT', message); - endVoiceConversation().then(); + endVoiceConversationRef.current?.(); saveCurrentMessages(); console.log('Voice chat error:', message); } - }, - // Handle audio level changes - (source, level) => { - if (source === 'microphone') { - inputAudioLevelRef.current = level; - } else { - outputAudioLevelRef.current = level; - } - const maxLevel = Math.max( - inputAudioLevelRef.current, - outputAudioLevelRef.current - ); - setAudioVolume(maxLevel); } ); @@ -211,7 +201,7 @@ function ChatScreen(): React.JSX.Element { return () => { voiceChatService.cleanup(); }; - }, [endVoiceConversation]); + }, []); // start new chat const startNewChat = useRef( @@ -300,7 +290,7 @@ function ChatScreen(): React.JSX.Element { // click from history setMessages([]); if (isNovaSonicRef.current) { - endVoiceConversation().then(); + endVoiceConversationRef.current?.(); } setIsLoadingMessages(true); const msg = getMessagesBySessionId(initialSessionId); @@ -324,7 +314,7 @@ function ChatScreen(): React.JSX.Element { }, 200); } } - }, [initialSessionId, mode, tapIndex, endVoiceConversation]); + }, [initialSessionId, mode, tapIndex]); // deleteChat listener useEffect(() => { @@ -709,11 +699,7 @@ function ChatScreen(): React.JSX.Element { } renderComposer={props => { if (isNovaSonic && mode === ChatMode.Text) { - return ( - - ); + return ; } // Default input box @@ -778,14 +764,14 @@ function ChatScreen(): React.JSX.Element { if (isNovaSonic) { saveCurrentVoiceSystemPrompt(prompt); if (chatStatus === ChatStatus.Running) { - endVoiceConversation().then(); + endVoiceConversationRef.current?.(); } } else { saveCurrentSystemPrompt(prompt); } }} onSwitchedToTextModel={() => { - endVoiceConversation().then(); + endVoiceConversationRef.current?.(); }} chatMode={modeRef.current} isShowSystemPrompt={showSystemPrompt} diff --git a/react-native/src/chat/component/AudioWaveformComponent.tsx b/react-native/src/chat/component/AudioWaveformComponent.tsx index a243665..bcd5c3e 100644 --- a/react-native/src/chat/component/AudioWaveformComponent.tsx +++ b/react-native/src/chat/component/AudioWaveformComponent.tsx @@ -1,4 +1,10 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { + useCallback, + useEffect, + useImperativeHandle, + useRef, + useState, +} from 'react'; import { View, StyleSheet, Dimensions } from 'react-native'; import Animated, { useSharedValue, @@ -8,157 +14,197 @@ import Animated, { Easing, } from 'react-native-reanimated'; import { isMac } from '../../App.tsx'; +import { voiceChatService } from '../service/VoiceChatService.ts'; -interface AudioWaveformProps { - volume?: number; // Volume level between 1-10 +export interface AudioWaveformRef { + resetAudioLevels: () => void; } const { width: screenWidth, height: screenHeight } = Dimensions.get('window'); const minWidth = screenWidth > screenHeight ? screenHeight : screenWidth; const isPad = minWidth > 434; -const AudioWaveformComponent: React.FC = ({ - volume = 1, -}) => { - const [colorOffset, setColorOffset] = useState(0); - const barCountRef = useRef(isMac || isPad ? 48 : 32); - const barValues = Array(barCountRef.current) - .fill(0) - // eslint-disable-next-line react-hooks/rules-of-hooks - .map(() => useSharedValue(0.3)); - - // Gradient colors from blue to green to purple - const gradientColors = [ - '#4158D0', - '#4B5EE8', - '#5564FF', - '#5F6CFF', - '#6975FF', - '#737EFF', - '#7D87FF', - '#8790FF', - '#90A0FF', - '#8BAFFF', - '#86BEFF', - '#80CDFF', - '#7ADCFF', - '#74EBFF', - '#6EFAFF', - '#68FFFC', - '#60F5F0', - '#58F0E0', - '#50EBD0', - '#48E6C0', - '#40E1B0', - '#38DCA0', - '#30D790', - '#29D280', - '#21CD70', - '#41D46C', - '#61DB68', - '#81E264', - '#A1E960', - '#B0ED5C', - '#C0F158', - '#D0F554', - '#C8F050', - '#BEC24C', - '#B49448', - '#AA6644', - '#A03840', - '#963A60', - '#8C3C80', - '#823EA0', - '#7840C0', - '#7E4CD8', - '#8458F0', - '#8A64FF', - '#9070FF', - '#967CFF', - '#9C88FF', - '#4158D0', - ]; - - // Color animation effect - updates every 500ms - useEffect(() => { - const colorAnimationInterval = setInterval(() => { - setColorOffset(prev => (prev + 1) % gradientColors.length); - }, 500); - - return () => clearInterval(colorAnimationInterval); - }, [gradientColors.length]); - - // Update waveform when volume changes - useEffect(() => { - // Special handling for volume=1 (silent or not recording) - if (volume === 1) { - barValues.forEach(bar => { - // Fixed low height for all bars - const minHeight = 0.05; - - bar.value = withTiming(minHeight, { - duration: 300, - easing: Easing.bezier(0.25, 0.1, 0.25, 1), +const AudioWaveformComponent = React.forwardRef( + (props, ref) => { + const [colorOffset, setColorOffset] = useState(0); + const barCountRef = useRef(isMac || isPad ? 48 : 32); + const barValues = Array(barCountRef.current) + .fill(0) + // eslint-disable-next-line react-hooks/rules-of-hooks + .map(() => useSharedValue(0.3)); + const inputAudioLevelRef = useRef(1); + const outputAudioLevelRef = useRef(1); + const [audioVolume, setAudioVolume] = useState(1); // Audio volume level (1-10) + + useEffect(() => { + // Set up voice chat service callbacks + voiceChatService.setOnAudioLevelCallbacks( + // Handle audio level changes + (source, level) => { + if (source === 'microphone') { + inputAudioLevelRef.current = level; + } else { + outputAudioLevelRef.current = level; + } + const maxLevel = Math.max( + inputAudioLevelRef.current, + outputAudioLevelRef.current + ); + setAudioVolume(maxLevel); + } + ); + }, []); + + // Add reset method for audio levels + const resetAudioLevels = useCallback(() => { + inputAudioLevelRef.current = 1; + outputAudioLevelRef.current = 1; + }, []); + + // Expose methods to parent component + useImperativeHandle( + ref, + () => ({ + resetAudioLevels, + }), + [resetAudioLevels] + ); + + // Gradient colors from blue to green to purple + const gradientColors = [ + '#4158D0', + '#4B5EE8', + '#5564FF', + '#5F6CFF', + '#6975FF', + '#737EFF', + '#7D87FF', + '#8790FF', + '#90A0FF', + '#8BAFFF', + '#86BEFF', + '#80CDFF', + '#7ADCFF', + '#74EBFF', + '#6EFAFF', + '#68FFFC', + '#60F5F0', + '#58F0E0', + '#50EBD0', + '#48E6C0', + '#40E1B0', + '#38DCA0', + '#30D790', + '#29D280', + '#21CD70', + '#41D46C', + '#61DB68', + '#81E264', + '#A1E960', + '#B0ED5C', + '#C0F158', + '#D0F554', + '#C8F050', + '#BEC24C', + '#B49448', + '#AA6644', + '#A03840', + '#963A60', + '#8C3C80', + '#823EA0', + '#7840C0', + '#7E4CD8', + '#8458F0', + '#8A64FF', + '#9070FF', + '#967CFF', + '#9C88FF', + '#4158D0', + ]; + + // Color animation effect - updates every 500ms + useEffect(() => { + const colorAnimationInterval = setInterval(() => { + setColorOffset(prev => (prev + 1) % gradientColors.length); + }, 500); + + return () => clearInterval(colorAnimationInterval); + }, [gradientColors.length]); + + // Update waveform when volume changes + useEffect(() => { + // Special handling for volume=1 (silent or not recording) + if (audioVolume === 1) { + barValues.forEach(bar => { + // Fixed low height for all bars + const minHeight = 0.05; + + bar.value = withTiming(minHeight, { + duration: 300, + easing: Easing.bezier(0.25, 0.1, 0.25, 1), + }); }); + return; + } + + // For volume > 1, animate based on volume level + const baseIntensity = audioVolume / 10; + + barValues.forEach((bar, index) => { + const centerEffect = + 1 - + Math.abs( + (index - barCountRef.current / 2) / (barCountRef.current / 2) + ) * + 0.5; + const randomHeight = + (Math.random() * 0.6 + 0.2) * baseIntensity * centerEffect; + const delay = index * 10; + + bar.value = withSequence( + withTiming(randomHeight, { + duration: 180 + delay, + easing: Easing.bezier(0.25, 0.1, 0.25, 1), + }), + withTiming(0.05 + Math.random() * 0.15 * baseIntensity, { + duration: 220 + delay, + easing: Easing.bezier(0.25, 0.1, 0.25, 1), + }) + ); }); - return; - } - - // For volume > 1, animate based on volume level - const baseIntensity = volume / 10; - - barValues.forEach((bar, index) => { - const centerEffect = - 1 - - Math.abs( - (index - barCountRef.current / 2) / (barCountRef.current / 2) - ) * - 0.5; - const randomHeight = - (Math.random() * 0.6 + 0.2) * baseIntensity * centerEffect; - const delay = index * 10; - - bar.value = withSequence( - withTiming(randomHeight, { - duration: 180 + delay, - easing: Easing.bezier(0.25, 0.1, 0.25, 1), - }), - withTiming(0.05 + Math.random() * 0.15 * baseIntensity, { - duration: 220 + delay, - easing: Easing.bezier(0.25, 0.1, 0.25, 1), - }) - ); - }); - }, [barValues, volume]); - - const animatedBarStyles = barValues.map(bar => - // eslint-disable-next-line react-hooks/rules-of-hooks - useAnimatedStyle(() => ({ - height: `${bar.value * 100}%`, - opacity: 0.7 + bar.value * 0.3, - })) - ); - - return ( - - - {barValues.map((_, index) => ( - - ))} + }, [barValues, audioVolume]); + + const animatedBarStyles = barValues.map(bar => + // eslint-disable-next-line react-hooks/rules-of-hooks + useAnimatedStyle(() => ({ + height: `${bar.value * 100}%`, + opacity: 0.7 + bar.value * 0.3, + })) + ); + + return ( + + + {barValues.map((_, index) => ( + + ))} + - - ); -}; + ); + } +); const styles = StyleSheet.create({ container: { diff --git a/react-native/src/chat/component/CustomSendComponent.tsx b/react-native/src/chat/component/CustomSendComponent.tsx index b94abb9..a68f4f5 100644 --- a/react-native/src/chat/component/CustomSendComponent.tsx +++ b/react-native/src/chat/component/CustomSendComponent.tsx @@ -44,7 +44,8 @@ const CustomSendComponent: React.FC = ({ ((text && text!.length > 0) || selectedFiles.length > 0 || chatStatus === ChatStatus.Running) && - !isNovaSonic; + !isNovaSonic && + !isShowLoading; } if (isShowSending) { return ( @@ -81,7 +82,7 @@ const CustomSendComponent: React.FC = ({ ); } else { - if (isNovaSonic && chatMode === ChatMode.Text) { + if ((isNovaSonic || isShowLoading) && chatMode === ChatMode.Text) { if (isShowLoading) { return ( diff --git a/react-native/src/chat/service/VoiceChatService.ts b/react-native/src/chat/service/VoiceChatService.ts index 9c86ac0..e98dc66 100644 --- a/react-native/src/chat/service/VoiceChatService.ts +++ b/react-native/src/chat/service/VoiceChatService.ts @@ -34,15 +34,22 @@ export class VoiceChatService { * Set callbacks for voice chat events * @param onTranscriptReceived Callback when transcript is received * @param onError Callback when error occurs - * @param onAudioLevelChanged Callback when audio level changes */ public setCallbacks( onTranscriptReceived?: (role: string, text: string) => void, - onError?: (message: string) => void, - onAudioLevelChanged?: (source: string, level: number) => void + onError?: (message: string) => void ) { this.onTranscriptReceivedCallback = onTranscriptReceived; this.onErrorCallback = onError; + } + + /** + * Set OnAudioLevelCallback for voice chat events + * @param onAudioLevelChanged Callback when audio level changes + */ + public setOnAudioLevelCallbacks( + onAudioLevelChanged?: (source: string, level: number) => void + ) { this.onAudioLevelChangedCallback = onAudioLevelChanged; } diff --git a/react-native/src/storage/Constants.ts b/react-native/src/storage/Constants.ts index 2eacb36..b538910 100644 --- a/react-native/src/storage/Constants.ts +++ b/react-native/src/storage/Constants.ts @@ -5,14 +5,19 @@ import { getDeepSeekApiKey, getOpenAIApiKey } from './StorageUtils.ts'; const RegionList = [ 'us-west-2', 'us-east-1', + 'us-east-2', 'ap-south-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', + 'ap-northeast-2', + 'ap-northeast-3', 'ca-central-1', 'eu-central-1', + 'eu-west-1', 'eu-west-2', 'eu-west-3', + 'eu-north-1', 'sa-east-1', ]; @@ -51,7 +56,11 @@ export const DeepSeekModels = [ }, ]; -export const BedrockThinkingModels = ['Claude 3.7 Sonnet']; +export const BedrockThinkingModels = [ + 'Claude 3.7 Sonnet', + 'Claude Sonnet 4', + 'Claude Opus 4', +]; export const BedrockVoiceModels = ['Nova Sonic']; export const DefaultTextModel = [ @@ -81,6 +90,14 @@ export const VoiceIDList = [ voiceName: 'Amy (British English)', voiceId: 'amy', }, + { + voiceName: 'Lupe (Spanish)', + voiceId: 'lupe', + }, + { + voiceName: 'Carlos (Spanish)', + voiceId: 'carlos', + }, ]; export const DefaultVoiceSystemPrompts = [ diff --git a/server/src/main.py b/server/src/main.py index 3ff8256..c8e7a8b 100644 --- a/server/src/main.py +++ b/server/src/main.py @@ -106,7 +106,9 @@ async def create_bedrock_command(request: ConverseRequest) -> tuple[boto3.client max_tokens = 4096 if model_id.startswith('meta.llama'): max_tokens = 2048 - if 'claude-3-7-sonnet' in model_id: + if 'deepseek.r1' in model_id or 'claude-opus-4' in model_id: + max_tokens = 32000 + if 'claude-3-7-sonnet' in model_id or 'claude-sonnet-4' in model_id: max_tokens = 64000 for message in request.messages: @@ -247,7 +249,10 @@ async def get_models(request: ModelsRequest, if ("TEXT" in model.get("outputModalities", []) and model.get("responseStreamingSupported")): if need_cross_region: - model_id = region.split("-")[0] + "." + model["modelId"] + region_prefix = region.split("-")[0] + if region_prefix == 'ap': + region_prefix = 'apac' + model_id = region_prefix + "." + model["modelId"] else: model_id = model["modelId"] text_model.append({