Add local Whisper speech input
This commit is contained in:
Ismail Ali
@@ -4,3 +4,4 @@ EXPO_PUBLIC_TTS_BASE_URL=http://localhost:3333
|
||||
EXPO_PUBLIC_TTS_VOICE=en-US-JennyNeural
|
||||
EXPO_PUBLIC_TTS_RATE=0.88
|
||||
EXPO_PUBLIC_TTS_PITCH=+0Hz
|
||||
EXPO_PUBLIC_STT_BASE_URL=http://localhost:3334
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -9,6 +9,8 @@ dist/
|
||||
web-build/
|
||||
expo-env.d.ts
|
||||
.tts-cache/
|
||||
.stt-uploads/
|
||||
.whisper-models/
|
||||
|
||||
# Native
|
||||
.kotlin/
|
||||
|
||||
20
README.md
20
README.md
@@ -109,6 +109,26 @@ npm run start
|
||||
|
||||
## Speech Input And Playback
|
||||
|
||||
Speech input can run through local Whisper on the laptop. The iPhone records audio, sends it to the local STT server, Whisper transcribes it to English text, and the app sends that text to Ollama automatically.
|
||||
|
||||
Start the STT server in a third terminal:
|
||||
|
||||
```bash
|
||||
npm run stt:start
|
||||
```
|
||||
|
||||
For Expo Go on iPhone, `.env` must point to the laptop IP:
|
||||
|
||||
```text
|
||||
EXPO_PUBLIC_STT_BASE_URL=http://192.168.10.33:3334
|
||||
```
|
||||
|
||||
The default local Whisper model is `tiny.en`. It is downloaded on first use and then runs locally without API costs. You can change it with:
|
||||
|
||||
```powershell
|
||||
$env:STT_MODEL="base.en"; npm run stt:start
|
||||
```
|
||||
|
||||
Playback uses a local MP3 TTS server on the laptop. AI replies are sent to the laptop, converted to an MP3 with a Microsoft neural English voice, and then played on the iPhone. This avoids the robotic iPhone system voice.
|
||||
|
||||
Start the TTS server in a second terminal:
|
||||
|
||||
5
app.json
5
app.json
@@ -15,7 +15,10 @@
|
||||
},
|
||||
"ios": {
|
||||
"supportsTablet": true,
|
||||
"bundleIdentifier": "com.englishaicoach.app"
|
||||
"bundleIdentifier": "com.englishaicoach.app",
|
||||
"infoPlist": {
|
||||
"NSMicrophoneUsageDescription": "English AI Coach uses the microphone to record your spoken English and transcribe it locally on your laptop."
|
||||
}
|
||||
},
|
||||
"android": {
|
||||
"adaptiveIcon": {
|
||||
|
||||
380
package-lock.json
generated
380
package-lock.json
generated
@@ -16,6 +16,8 @@
|
||||
"expo-status-bar": "~3.0.9",
|
||||
"express": "^5.2.1",
|
||||
"msedge-tts": "^2.0.5",
|
||||
"multer": "^2.1.1",
|
||||
"nodejs-whisper": "^0.3.0",
|
||||
"react": "19.1.0",
|
||||
"react-native": "0.81.5",
|
||||
"react-native-safe-area-context": "~5.6.0",
|
||||
@@ -2712,6 +2714,41 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@nodelib/fs.stat": "2.0.5",
|
||||
"run-parallel": "^1.1.9"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.stat": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
|
||||
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.walk": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
|
||||
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@nodelib/fs.scandir": "2.1.5",
|
||||
"fastq": "^1.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-native/assets-registry": {
|
||||
"version": "0.81.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.81.5.tgz",
|
||||
@@ -3384,6 +3421,12 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/append-field": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
|
||||
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
||||
@@ -3920,6 +3963,17 @@
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
|
||||
"dependencies": {
|
||||
"streamsearch": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
@@ -4259,6 +4313,21 @@
|
||||
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/concat-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.0.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/connect": {
|
||||
"version": "3.7.0",
|
||||
"resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz",
|
||||
@@ -4674,6 +4743,53 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/execa": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
|
||||
"integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cross-spawn": "^7.0.3",
|
||||
"get-stream": "^6.0.0",
|
||||
"human-signals": "^2.1.0",
|
||||
"is-stream": "^2.0.0",
|
||||
"merge-stream": "^2.0.0",
|
||||
"npm-run-path": "^4.0.1",
|
||||
"onetime": "^5.1.2",
|
||||
"signal-exit": "^3.0.3",
|
||||
"strip-final-newline": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sindresorhus/execa?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/execa/node_modules/mimic-fn": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
|
||||
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/execa/node_modules/onetime": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
|
||||
"integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mimic-fn": "^2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/expo": {
|
||||
"version": "54.0.34",
|
||||
"resolved": "https://registry.npmjs.org/expo/-/expo-54.0.34.tgz",
|
||||
@@ -5510,12 +5626,37 @@
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-glob": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
|
||||
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@nodelib/fs.stat": "^2.0.2",
|
||||
"@nodelib/fs.walk": "^1.2.3",
|
||||
"glob-parent": "^5.1.2",
|
||||
"merge2": "^1.3.0",
|
||||
"micromatch": "^4.0.8"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-json-stable-stringify": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
|
||||
"integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fastq": {
|
||||
"version": "1.20.1",
|
||||
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
|
||||
"integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"reusify": "^1.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/fb-watchman": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz",
|
||||
@@ -5760,6 +5901,18 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-stream": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
|
||||
"integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/getenv": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/getenv/-/getenv-2.0.0.tgz",
|
||||
@@ -5786,6 +5939,18 @@
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-parent": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
|
||||
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"is-glob": "^4.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
@@ -5927,6 +6092,15 @@
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/human-signals": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
|
||||
"integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=10.17.0"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
|
||||
@@ -6073,6 +6247,15 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-extglob": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
|
||||
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
@@ -6082,6 +6265,18 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/is-glob": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
|
||||
"integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-extglob": "^2.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-number": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
|
||||
@@ -6097,6 +6292,18 @@
|
||||
"integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-stream": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
|
||||
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-wsl": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
|
||||
@@ -7040,6 +7247,15 @@
|
||||
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/merge2": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
|
||||
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/metro": {
|
||||
"version": "0.83.3",
|
||||
"resolved": "https://registry.npmjs.org/metro/-/metro-0.83.3.tgz",
|
||||
@@ -7573,6 +7789,47 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/multer": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz",
|
||||
"integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
"concat-stream": "^2.0.0",
|
||||
"type-is": "^1.6.18"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.16.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/media-typer": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
"integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/multer/node_modules/type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
|
||||
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"media-typer": "0.3.0",
|
||||
"mime-types": "~2.1.24"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
@@ -7638,6 +7895,19 @@
|
||||
"integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nodejs-whisper": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/nodejs-whisper/-/nodejs-whisper-0.3.0.tgz",
|
||||
"integrity": "sha512-fIj1Aw6mKevfWKW2wuoFEZuYDRAJPM+L9VWdceGgji3sU8+QKoO8pI30L3+kCYVCCssGOXRzU5qQ0eZdRhfMyg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"readline-sync": "^1.4.10",
|
||||
"shelljs": "^0.10.0"
|
||||
},
|
||||
"bin": {
|
||||
"nodejs-whisper": "dist/downloadModel.js"
|
||||
}
|
||||
},
|
||||
"node_modules/normalize-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
@@ -7662,6 +7932,18 @@
|
||||
"node": "^16.14.0 || >=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/npm-run-path": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
|
||||
"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"path-key": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/nullthrows": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz",
|
||||
@@ -8183,6 +8465,26 @@
|
||||
"inherits": "~2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/queue-microtask": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
|
||||
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
@@ -8464,6 +8766,15 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/readline-sync": {
|
||||
"version": "1.4.10",
|
||||
"resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
|
||||
"integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/regenerate": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz",
|
||||
@@ -8621,6 +8932,16 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/reusify": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
|
||||
"integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"iojs": ">=1.0.0",
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
@@ -8702,6 +9023,29 @@
|
||||
"node": ">= 18"
|
||||
}
|
||||
},
|
||||
"node_modules/run-parallel": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"queue-microtask": "^1.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -8905,6 +9249,19 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/shelljs": {
|
||||
"version": "0.10.0",
|
||||
"resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.10.0.tgz",
|
||||
"integrity": "sha512-Jex+xw5Mg2qMZL3qnzXIfaxEtBaC4n7xifqaqtrZDdlheR70OGkydrPJWT0V1cA1k3nanC86x9FwAmQl6w3Klw==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"execa": "^5.1.1",
|
||||
"fast-glob": "^3.3.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/side-channel": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
|
||||
@@ -9146,6 +9503,14 @@
|
||||
"node": ">= 0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/strict-uri-encode": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz",
|
||||
@@ -9190,6 +9555,15 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-final-newline": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
|
||||
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-json-comments": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
|
||||
@@ -9584,6 +9958,12 @@
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/typedarray": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
|
||||
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
"ios": "expo start --ios",
|
||||
"web": "expo start --web",
|
||||
"prebuild:ios": "expo prebuild --platform ios",
|
||||
"tts:start": "node tools/tts-server.mjs"
|
||||
"tts:start": "node tools/tts-server.mjs",
|
||||
"stt:start": "node tools/stt-server.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@react-navigation/native": "^7.2.2",
|
||||
@@ -19,6 +20,8 @@
|
||||
"expo-status-bar": "~3.0.9",
|
||||
"express": "^5.2.1",
|
||||
"msedge-tts": "^2.0.5",
|
||||
"multer": "^2.1.1",
|
||||
"nodejs-whisper": "^0.3.0",
|
||||
"react": "19.1.0",
|
||||
"react-native": "0.81.5",
|
||||
"react-native-safe-area-context": "~5.6.0",
|
||||
|
||||
3
src/config/stt.ts
Normal file
3
src/config/stt.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export const sttConfig = {
|
||||
baseUrl: process.env.EXPO_PUBLIC_STT_BASE_URL ?? '',
|
||||
};
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { NativeStackScreenProps } from '@react-navigation/native-stack';
|
||||
import { useState } from 'react';
|
||||
import { Audio } from 'expo-av';
|
||||
import { useRef, useState } from 'react';
|
||||
import { FlatList, KeyboardAvoidingView, Platform, StyleSheet, Text, TextInput, View } from 'react-native';
|
||||
import { SafeAreaView } from 'react-native-safe-area-context';
|
||||
|
||||
@@ -8,12 +9,14 @@ import { PrimaryButton } from '../components/PrimaryButton';
|
||||
import type { RootStackParamList } from '../navigation/types';
|
||||
import { getMockAiReply } from '../services/mockAiCoach';
|
||||
import { speakText, stopSpeaking } from '../services/speechService';
|
||||
import { transcribeAudio } from '../services/sttService';
|
||||
import { colors } from '../theme/colors';
|
||||
|
||||
type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>;
|
||||
|
||||
export function ChatScreen({ route }: ChatScreenProps) {
|
||||
const { level } = route.params;
|
||||
const recordingRef = useRef<Audio.Recording | null>(null);
|
||||
const [input, setInput] = useState('I go yesterday to work');
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([
|
||||
{
|
||||
@@ -23,13 +26,23 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
||||
},
|
||||
]);
|
||||
const [isSending, setIsSending] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
|
||||
async function sendMessage() {
|
||||
if (isSending || !input.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: input.trim() };
|
||||
await sendText(input.trim());
|
||||
}
|
||||
|
||||
async function sendText(text: string) {
|
||||
if (isSending || !text.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: text.trim() };
|
||||
setMessages((current) => [...current, userMessage]);
|
||||
setInput('');
|
||||
setIsSending(true);
|
||||
@@ -37,10 +50,73 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
||||
// Uses Ollama when available and falls back to local examples when offline.
|
||||
const reply = await getMockAiReply(userMessage.text, level);
|
||||
setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]);
|
||||
speakText(reply);
|
||||
void speakText(reply).catch(() => undefined);
|
||||
setIsSending(false);
|
||||
}
|
||||
|
||||
async function toggleRecording() {
|
||||
if (isRecording) {
|
||||
await stopRecordingAndSend();
|
||||
return;
|
||||
}
|
||||
|
||||
await startRecording();
|
||||
}
|
||||
|
||||
async function startRecording() {
|
||||
const permission = await Audio.requestPermissionsAsync();
|
||||
|
||||
if (!permission.granted) {
|
||||
setInput('Microphone permission is required to record your voice.');
|
||||
return;
|
||||
}
|
||||
|
||||
await Audio.setAudioModeAsync({
|
||||
allowsRecordingIOS: true,
|
||||
playsInSilentModeIOS: true,
|
||||
});
|
||||
|
||||
const recording = new Audio.Recording();
|
||||
await recording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY);
|
||||
await recording.startAsync();
|
||||
recordingRef.current = recording;
|
||||
setIsRecording(true);
|
||||
}
|
||||
|
||||
async function stopRecordingAndSend() {
|
||||
const recording = recordingRef.current;
|
||||
|
||||
if (!recording) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIsRecording(false);
|
||||
setIsTranscribing(true);
|
||||
recordingRef.current = null;
|
||||
|
||||
await recording.stopAndUnloadAsync();
|
||||
await Audio.setAudioModeAsync({ allowsRecordingIOS: false });
|
||||
|
||||
const uri = recording.getURI();
|
||||
if (!uri) {
|
||||
setIsTranscribing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const transcript = await transcribeAudio(uri);
|
||||
setInput(transcript);
|
||||
|
||||
if (transcript) {
|
||||
await sendText(transcript);
|
||||
}
|
||||
} catch {
|
||||
setInput('I could not transcribe the recording. Please try again.');
|
||||
} finally {
|
||||
setIsTranscribing(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<SafeAreaView style={styles.safeArea} edges={['bottom']}>
|
||||
<KeyboardAvoidingView
|
||||
@@ -65,8 +141,13 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
||||
value={input}
|
||||
/>
|
||||
<Text style={styles.dictationHint}>
|
||||
Tip: tap this field and use the iPhone keyboard microphone to dictate your sentence.
|
||||
Use Record to send your voice through local Whisper, or use the keyboard microphone for iPhone dictation.
|
||||
</Text>
|
||||
<PrimaryButton
|
||||
label={isRecording ? 'Stop recording and send' : isTranscribing ? 'Transcribing...' : 'Record voice'}
|
||||
variant={isRecording ? 'primary' : 'secondary'}
|
||||
onPress={toggleRecording}
|
||||
/>
|
||||
<View style={styles.actionRow}>
|
||||
<View style={styles.actionItem}>
|
||||
<PrimaryButton label={isSending ? 'Thinking...' : 'Send'} onPress={sendMessage} />
|
||||
@@ -81,7 +162,7 @@ export function ChatScreen({ route }: ChatScreenProps) {
|
||||
onPress={() => {
|
||||
const lastAiMessage = [...messages].reverse().find((message) => message.role === 'assistant');
|
||||
if (lastAiMessage) {
|
||||
speakText(lastAiMessage.text);
|
||||
void speakText(lastAiMessage.text).catch(() => undefined);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
|
||||
26
src/services/sttService.ts
Normal file
26
src/services/sttService.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { sttConfig } from '../config/stt';
|
||||
|
||||
export async function transcribeAudio(uri: string) {
|
||||
if (!sttConfig.baseUrl) {
|
||||
throw new Error('STT server URL is not configured.');
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('audio', {
|
||||
uri,
|
||||
name: 'speech.m4a',
|
||||
type: 'audio/mp4',
|
||||
} as unknown as Blob);
|
||||
|
||||
const response = await fetch(`${sttConfig.baseUrl}/transcribe`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`STT request failed with ${response.status}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { text?: string };
|
||||
return data.text?.trim() ?? '';
|
||||
}
|
||||
88
tools/stt-server.mjs
Normal file
88
tools/stt-server.mjs
Normal file
@@ -0,0 +1,88 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import cors from 'cors';
|
||||
import express from 'express';
|
||||
import multer from 'multer';
|
||||
import { nodewhisper } from 'nodejs-whisper';
|
||||
|
||||
const app = express();
|
||||
const port = Number(process.env.STT_PORT ?? 3334);
|
||||
const host = process.env.STT_HOST ?? '0.0.0.0';
|
||||
const uploadDir = path.resolve(process.cwd(), '.stt-uploads');
|
||||
const modelRootPath = path.resolve(process.cwd(), '.whisper-models');
|
||||
const modelName = process.env.STT_MODEL ?? 'tiny.en';
|
||||
|
||||
fs.mkdirSync(uploadDir, { recursive: true });
|
||||
fs.mkdirSync(modelRootPath, { recursive: true });
|
||||
|
||||
const upload = multer({ dest: uploadDir });
|
||||
|
||||
app.use(cors());
|
||||
|
||||
app.get('/health', (_request, response) => {
|
||||
response.json({ ok: true, model: modelName });
|
||||
});
|
||||
|
||||
app.post('/transcribe', upload.single('audio'), async (request, response) => {
|
||||
if (!request.file) {
|
||||
response.status(400).json({ error: 'Missing audio file' });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const text = await nodewhisper(request.file.path, {
|
||||
modelName,
|
||||
autoDownloadModelName: modelName,
|
||||
modelRootPath,
|
||||
removeWavFileAfterTranscription: true,
|
||||
withCuda: false,
|
||||
logger: quietLogger,
|
||||
whisperOptions: {
|
||||
outputInText: false,
|
||||
outputInSrt: false,
|
||||
outputInVtt: false,
|
||||
outputInCsv: false,
|
||||
outputInJson: false,
|
||||
outputInJsonFull: false,
|
||||
outputInLrc: false,
|
||||
outputInWords: false,
|
||||
translateToEnglish: false,
|
||||
splitOnWord: true,
|
||||
noGpu: true,
|
||||
},
|
||||
});
|
||||
|
||||
response.json({ text: cleanTranscript(text) });
|
||||
} catch (error) {
|
||||
response.status(500).json({ error: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
safeUnlink(request.file.path);
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(port, host, () => {
|
||||
console.log(`STT server listening on http://${host}:${port}`);
|
||||
console.log(`Whisper model: ${modelName}`);
|
||||
});
|
||||
|
||||
const quietLogger = {
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
log: () => undefined,
|
||||
warn: console.warn,
|
||||
error: console.error,
|
||||
};
|
||||
|
||||
function cleanTranscript(value) {
|
||||
return String(value)
|
||||
.replace(/\[[^\]]*\]/g, ' ')
|
||||
.replace(/\([^)]*\)/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function safeUnlink(filePath) {
|
||||
if (filePath && fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user