Add speech playback for coach replies

This commit is contained in:
Ismail Ali
2026-04-28 15:28:40 +02:00
parent f288135378
commit 63d48a5b59
5 changed files with 72 additions and 1 deletions

View File

@@ -9,6 +9,7 @@ Das Projekt ist bewusst schlank gehalten: React Native, Expo, TypeScript und Rea
- HomeScreen mit Fortschrittskarte und Mock Daily Reminder - HomeScreen mit Fortschrittskarte und Mock Daily Reminder
- Level-Auswahl fuer A1, A2, B1 und B2 - Level-Auswahl fuer A1, A2, B1 und B2
- ChatScreen mit einfacher User- und AI-Nachrichtenansicht - ChatScreen mit einfacher User- und AI-Nachrichtenansicht
- AI-Antworten koennen auf dem iPhone vorgelesen werden
- Mock KI-Service fuer Beispielkorrekturen - Mock KI-Service fuer Beispielkorrekturen
- TypeScript-typisierte Navigation - TypeScript-typisierte Navigation
- Struktur fuer spaetere Services, Hooks und Utilities - Struktur fuer spaetere Services, Hooks und Utilities
@@ -32,6 +33,8 @@ Good try. Correct: "I went to work yesterday."
- React Navigation - React Navigation
- Geplant: Ollama API fuer lokale KI - Geplant: Ollama API fuer lokale KI
- Geplant: Speech-to-Text und Text-to-Speech - Geplant: Speech-to-Text und Text-to-Speech
- Text-to-Speech mit `expo-speech`
- Spracheingabe aktuell ueber iPhone-Diktierfunktion in der Tastatur
- Geplant: Push Notifications fuer Daily Reminder - Geplant: Push Notifications fuer Daily Reminder
## Projektstruktur ## Projektstruktur
@@ -104,6 +107,19 @@ App starten:
npm run start npm run start
``` ```
## Spracheingabe und Vorlesen
Vorlesen funktioniert direkt in Expo Go. AI-Antworten werden automatisch ueber den iPhone-Lautsprecher vorgelesen. Im Chat gibt es ausserdem Buttons fuer `Letzte Antwort vorlesen` und `Stop`.
Spracheingabe funktioniert aktuell ueber die iPhone-Tastatur:
- Chat-Eingabefeld antippen
- Mikrofon-Symbol auf der iPhone-Tastatur druecken
- Englisch sprechen
- Text pruefen und senden
Echtes In-App Speech-to-Text mit eigenem Mikrofon-Button ist fuer spaeter geplant. Dafuer braucht die App entweder ein natives iOS-Modul nach `expo prebuild` oder eine Transkriptions-API wie Whisper.
iOS Preview starten: iOS Preview starten:
```bash ```bash

10
package-lock.json generated
View File

@@ -11,6 +11,7 @@
"@react-navigation/native": "^7.2.2", "@react-navigation/native": "^7.2.2",
"@react-navigation/native-stack": "^7.14.12", "@react-navigation/native-stack": "^7.14.12",
"expo": "~54.0.33", "expo": "~54.0.33",
"expo-speech": "~14.0.8",
"expo-status-bar": "~3.0.9", "expo-status-bar": "~3.0.9",
"react": "19.1.0", "react": "19.1.0",
"react-native": "0.81.5", "react-native": "0.81.5",
@@ -4620,6 +4621,15 @@
"node": ">=20.16.0" "node": ">=20.16.0"
} }
}, },
"node_modules/expo-speech": {
"version": "14.0.8",
"resolved": "https://registry.npmjs.org/expo-speech/-/expo-speech-14.0.8.tgz",
"integrity": "sha512-UjBFCFv58nutlLw92L7kUS0ZjbOOfaTdiEv/HbjvMrT6BfldoOLLBZbaEcEhDdZK36NY/kass0Kzxk+co6vxSQ==",
"license": "MIT",
"peerDependencies": {
"expo": "*"
}
},
"node_modules/expo-status-bar": { "node_modules/expo-status-bar": {
"version": "3.0.9", "version": "3.0.9",
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-3.0.9.tgz", "resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-3.0.9.tgz",

View File

@@ -13,6 +13,7 @@
"@react-navigation/native": "^7.2.2", "@react-navigation/native": "^7.2.2",
"@react-navigation/native-stack": "^7.14.12", "@react-navigation/native-stack": "^7.14.12",
"expo": "~54.0.33", "expo": "~54.0.33",
"expo-speech": "~14.0.8",
"expo-status-bar": "~3.0.9", "expo-status-bar": "~3.0.9",
"react": "19.1.0", "react": "19.1.0",
"react-native": "0.81.5", "react-native": "0.81.5",

View File

@@ -7,6 +7,7 @@ import { ChatBubble, type ChatMessage } from '../components/ChatBubble';
import { PrimaryButton } from '../components/PrimaryButton'; import { PrimaryButton } from '../components/PrimaryButton';
import type { RootStackParamList } from '../navigation/types'; import type { RootStackParamList } from '../navigation/types';
import { getMockAiReply } from '../services/mockAiCoach'; import { getMockAiReply } from '../services/mockAiCoach';
import { speakText, stopSpeaking } from '../services/speechService';
import { colors } from '../theme/colors'; import { colors } from '../theme/colors';
type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>; type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>;
@@ -36,6 +37,7 @@ export function ChatScreen({ route }: ChatScreenProps) {
// This mock can later be replaced by Ollama or another AI API client. // This mock can later be replaced by Ollama or another AI API client.
const reply = await getMockAiReply(userMessage.text, level); const reply = await getMockAiReply(userMessage.text, level);
setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]); setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]);
speakText(reply);
setIsSending(false); setIsSending(false);
} }
@@ -62,8 +64,25 @@ export function ChatScreen({ route }: ChatScreenProps) {
style={styles.input} style={styles.input}
value={input} value={input}
/> />
<View style={styles.actionRow}>
<View style={styles.actionItem}>
<PrimaryButton label={isSending ? 'Denke...' : 'Senden'} onPress={sendMessage} /> <PrimaryButton label={isSending ? 'Denke...' : 'Senden'} onPress={sendMessage} />
</View> </View>
<View style={styles.actionItem}>
<PrimaryButton label="Stop" variant="secondary" onPress={stopSpeaking} />
</View>
</View>
<PrimaryButton
label="Letzte Antwort vorlesen"
variant="secondary"
onPress={() => {
const lastAiMessage = [...messages].reverse().find((message) => message.role === 'assistant');
if (lastAiMessage) {
speakText(lastAiMessage.text);
}
}}
/>
</View>
</KeyboardAvoidingView> </KeyboardAvoidingView>
</SafeAreaView> </SafeAreaView>
); );
@@ -86,6 +105,13 @@ const styles = StyleSheet.create({
gap: 12, gap: 12,
padding: 14, padding: 14,
}, },
actionRow: {
flexDirection: 'row',
gap: 10,
},
actionItem: {
flex: 1,
},
input: { input: {
backgroundColor: colors.surfaceRaised, backgroundColor: colors.surfaceRaised,
borderColor: colors.border, borderColor: colors.border,

View File

@@ -0,0 +1,18 @@
import * as Speech from 'expo-speech';
export function speakText(text: string) {
Speech.stop();
Speech.speak(stripThinkingBlocks(text), {
language: 'en-US',
pitch: 1,
rate: 0.9,
});
}
export function stopSpeaking() {
Speech.stop();
}
function stripThinkingBlocks(text: string) {
return text.replace(/<think>[\s\S]*?<\/think>/gi, '').trim();
}