Add speech playback for coach replies
This commit is contained in:
16
README.md
16
README.md
@@ -9,6 +9,7 @@ Das Projekt ist bewusst schlank gehalten: React Native, Expo, TypeScript und Rea
|
|||||||
- HomeScreen mit Fortschrittskarte und Mock Daily Reminder
|
- HomeScreen mit Fortschrittskarte und Mock Daily Reminder
|
||||||
- Level-Auswahl fuer A1, A2, B1 und B2
|
- Level-Auswahl fuer A1, A2, B1 und B2
|
||||||
- ChatScreen mit einfacher User- und AI-Nachrichtenansicht
|
- ChatScreen mit einfacher User- und AI-Nachrichtenansicht
|
||||||
|
- AI-Antworten koennen auf dem iPhone vorgelesen werden
|
||||||
- Mock KI-Service fuer Beispielkorrekturen
|
- Mock KI-Service fuer Beispielkorrekturen
|
||||||
- TypeScript-typisierte Navigation
|
- TypeScript-typisierte Navigation
|
||||||
- Struktur fuer spaetere Services, Hooks und Utilities
|
- Struktur fuer spaetere Services, Hooks und Utilities
|
||||||
@@ -32,6 +33,8 @@ Good try. Correct: "I went to work yesterday."
|
|||||||
- React Navigation
|
- React Navigation
|
||||||
- Geplant: Ollama API fuer lokale KI
|
- Geplant: Ollama API fuer lokale KI
|
||||||
- Geplant: Speech-to-Text und Text-to-Speech
|
- Geplant: Speech-to-Text und Text-to-Speech
|
||||||
|
- Text-to-Speech mit `expo-speech`
|
||||||
|
- Spracheingabe aktuell ueber iPhone-Diktierfunktion in der Tastatur
|
||||||
- Geplant: Push Notifications fuer Daily Reminder
|
- Geplant: Push Notifications fuer Daily Reminder
|
||||||
|
|
||||||
## Projektstruktur
|
## Projektstruktur
|
||||||
@@ -104,6 +107,19 @@ App starten:
|
|||||||
npm run start
|
npm run start
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Spracheingabe und Vorlesen
|
||||||
|
|
||||||
|
Vorlesen funktioniert direkt in Expo Go. AI-Antworten werden automatisch ueber den iPhone-Lautsprecher vorgelesen. Im Chat gibt es ausserdem Buttons fuer `Letzte Antwort vorlesen` und `Stop`.
|
||||||
|
|
||||||
|
Spracheingabe funktioniert aktuell ueber die iPhone-Tastatur:
|
||||||
|
|
||||||
|
- Chat-Eingabefeld antippen
|
||||||
|
- Mikrofon-Symbol auf der iPhone-Tastatur druecken
|
||||||
|
- Englisch sprechen
|
||||||
|
- Text pruefen und senden
|
||||||
|
|
||||||
|
Echtes In-App Speech-to-Text mit eigenem Mikrofon-Button ist fuer spaeter geplant. Dafuer braucht die App entweder ein natives iOS-Modul nach `expo prebuild` oder eine Transkriptions-API wie Whisper.
|
||||||
|
|
||||||
iOS Preview starten:
|
iOS Preview starten:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
10
package-lock.json
generated
10
package-lock.json
generated
@@ -11,6 +11,7 @@
|
|||||||
"@react-navigation/native": "^7.2.2",
|
"@react-navigation/native": "^7.2.2",
|
||||||
"@react-navigation/native-stack": "^7.14.12",
|
"@react-navigation/native-stack": "^7.14.12",
|
||||||
"expo": "~54.0.33",
|
"expo": "~54.0.33",
|
||||||
|
"expo-speech": "~14.0.8",
|
||||||
"expo-status-bar": "~3.0.9",
|
"expo-status-bar": "~3.0.9",
|
||||||
"react": "19.1.0",
|
"react": "19.1.0",
|
||||||
"react-native": "0.81.5",
|
"react-native": "0.81.5",
|
||||||
@@ -4620,6 +4621,15 @@
|
|||||||
"node": ">=20.16.0"
|
"node": ">=20.16.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/expo-speech": {
|
||||||
|
"version": "14.0.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/expo-speech/-/expo-speech-14.0.8.tgz",
|
||||||
|
"integrity": "sha512-UjBFCFv58nutlLw92L7kUS0ZjbOOfaTdiEv/HbjvMrT6BfldoOLLBZbaEcEhDdZK36NY/kass0Kzxk+co6vxSQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"peerDependencies": {
|
||||||
|
"expo": "*"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/expo-status-bar": {
|
"node_modules/expo-status-bar": {
|
||||||
"version": "3.0.9",
|
"version": "3.0.9",
|
||||||
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-3.0.9.tgz",
|
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-3.0.9.tgz",
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
"@react-navigation/native": "^7.2.2",
|
"@react-navigation/native": "^7.2.2",
|
||||||
"@react-navigation/native-stack": "^7.14.12",
|
"@react-navigation/native-stack": "^7.14.12",
|
||||||
"expo": "~54.0.33",
|
"expo": "~54.0.33",
|
||||||
|
"expo-speech": "~14.0.8",
|
||||||
"expo-status-bar": "~3.0.9",
|
"expo-status-bar": "~3.0.9",
|
||||||
"react": "19.1.0",
|
"react": "19.1.0",
|
||||||
"react-native": "0.81.5",
|
"react-native": "0.81.5",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { ChatBubble, type ChatMessage } from '../components/ChatBubble';
|
|||||||
import { PrimaryButton } from '../components/PrimaryButton';
|
import { PrimaryButton } from '../components/PrimaryButton';
|
||||||
import type { RootStackParamList } from '../navigation/types';
|
import type { RootStackParamList } from '../navigation/types';
|
||||||
import { getMockAiReply } from '../services/mockAiCoach';
|
import { getMockAiReply } from '../services/mockAiCoach';
|
||||||
|
import { speakText, stopSpeaking } from '../services/speechService';
|
||||||
import { colors } from '../theme/colors';
|
import { colors } from '../theme/colors';
|
||||||
|
|
||||||
type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>;
|
type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>;
|
||||||
@@ -36,6 +37,7 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
|||||||
// This mock can later be replaced by Ollama or another AI API client.
|
// This mock can later be replaced by Ollama or another AI API client.
|
||||||
const reply = await getMockAiReply(userMessage.text, level);
|
const reply = await getMockAiReply(userMessage.text, level);
|
||||||
setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]);
|
setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]);
|
||||||
|
speakText(reply);
|
||||||
setIsSending(false);
|
setIsSending(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,8 +64,25 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
|||||||
style={styles.input}
|
style={styles.input}
|
||||||
value={input}
|
value={input}
|
||||||
/>
|
/>
|
||||||
|
<View style={styles.actionRow}>
|
||||||
|
<View style={styles.actionItem}>
|
||||||
<PrimaryButton label={isSending ? 'Denke...' : 'Senden'} onPress={sendMessage} />
|
<PrimaryButton label={isSending ? 'Denke...' : 'Senden'} onPress={sendMessage} />
|
||||||
</View>
|
</View>
|
||||||
|
<View style={styles.actionItem}>
|
||||||
|
<PrimaryButton label="Stop" variant="secondary" onPress={stopSpeaking} />
|
||||||
|
</View>
|
||||||
|
</View>
|
||||||
|
<PrimaryButton
|
||||||
|
label="Letzte Antwort vorlesen"
|
||||||
|
variant="secondary"
|
||||||
|
onPress={() => {
|
||||||
|
const lastAiMessage = [...messages].reverse().find((message) => message.role === 'assistant');
|
||||||
|
if (lastAiMessage) {
|
||||||
|
speakText(lastAiMessage.text);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</View>
|
||||||
</KeyboardAvoidingView>
|
</KeyboardAvoidingView>
|
||||||
</SafeAreaView>
|
</SafeAreaView>
|
||||||
);
|
);
|
||||||
@@ -86,6 +105,13 @@ const styles = StyleSheet.create({
|
|||||||
gap: 12,
|
gap: 12,
|
||||||
padding: 14,
|
padding: 14,
|
||||||
},
|
},
|
||||||
|
actionRow: {
|
||||||
|
flexDirection: 'row',
|
||||||
|
gap: 10,
|
||||||
|
},
|
||||||
|
actionItem: {
|
||||||
|
flex: 1,
|
||||||
|
},
|
||||||
input: {
|
input: {
|
||||||
backgroundColor: colors.surfaceRaised,
|
backgroundColor: colors.surfaceRaised,
|
||||||
borderColor: colors.border,
|
borderColor: colors.border,
|
||||||
|
|||||||
18
src/services/speechService.ts
Normal file
18
src/services/speechService.ts
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
import * as Speech from 'expo-speech';
|
||||||
|
|
||||||
|
export function speakText(text: string) {
|
||||||
|
Speech.stop();
|
||||||
|
Speech.speak(stripThinkingBlocks(text), {
|
||||||
|
language: 'en-US',
|
||||||
|
pitch: 1,
|
||||||
|
rate: 0.9,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stopSpeaking() {
|
||||||
|
Speech.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripThinkingBlocks(text: string) {
|
||||||
|
return text.replace(/<think>[\s\S]*?<\/think>/gi, '').trim();
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user