mirror of
https://github.com/jeremysrand/ListenerApp.git
synced 2024-06-03 06:29:32 +00:00
Add code to pace the text to the GS to reduce the amount of re-typing required.
This commit is contained in:
parent
241190600e
commit
b2c2cbd587
|
@ -52,7 +52,7 @@ struct GSView: View {
|
||||||
speechForwarder.connect(destination: ipAddress)
|
speechForwarder.connect(destination: ipAddress)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.disabled(false)
|
.disabled(speechForwarder.connecting)
|
||||||
.buttonStyle(GSButtonStyle())
|
.buttonStyle(GSButtonStyle())
|
||||||
|
|
||||||
Button(speechForwarder.listening ?
|
Button(speechForwarder.listening ?
|
||||||
|
@ -60,7 +60,7 @@ struct GSView: View {
|
||||||
"\(Image(systemName: "ear.and.waveform")) Listen and Send Text") {
|
"\(Image(systemName: "ear.and.waveform")) Listen and Send Text") {
|
||||||
speechForwarder.listen()
|
speechForwarder.listen()
|
||||||
}
|
}
|
||||||
.disabled(!speechForwarder.connected)
|
.disabled((!speechForwarder.connected) || (!speechForwarder.listening && speechForwarder.sending))
|
||||||
.buttonStyle(GSButtonStyle())
|
.buttonStyle(GSButtonStyle())
|
||||||
}
|
}
|
||||||
.fixedSize(horizontal: true, vertical: false)
|
.fixedSize(horizontal: true, vertical: false)
|
||||||
|
|
|
@ -19,7 +19,7 @@
|
||||||
<key>CFBundleShortVersionString</key>
|
<key>CFBundleShortVersionString</key>
|
||||||
<string>1.0</string>
|
<string>1.0</string>
|
||||||
<key>CFBundleVersion</key>
|
<key>CFBundleVersion</key>
|
||||||
<string>390</string>
|
<string>434</string>
|
||||||
<key>LSApplicationCategoryType</key>
|
<key>LSApplicationCategoryType</key>
|
||||||
<string>public.app-category.utilities</string>
|
<string>public.app-category.utilities</string>
|
||||||
<key>LSRequiresIPhoneOS</key>
|
<key>LSRequiresIPhoneOS</key>
|
||||||
|
|
|
@ -12,10 +12,13 @@ import Speech
|
||||||
class SpeechForwarder : ObservableObject {
|
class SpeechForwarder : ObservableObject {
|
||||||
@Published var listening = false
|
@Published var listening = false
|
||||||
@Published var connected = false
|
@Published var connected = false
|
||||||
|
@Published var connecting = false
|
||||||
@Published var textHeard = ""
|
@Published var textHeard = ""
|
||||||
|
@Published var sending = false
|
||||||
|
|
||||||
let LISTEN_STATE_MSG = 1
|
let LISTEN_STATE_MSG = 1
|
||||||
let LISTEN_TEXT_MSG = 2
|
let LISTEN_TEXT_MSG = 2
|
||||||
|
let LISTEN_SEND_MORE = 3
|
||||||
|
|
||||||
let port = 19026
|
let port = 19026
|
||||||
private var client: TCPClient?
|
private var client: TCPClient?
|
||||||
|
@ -30,19 +33,31 @@ class SpeechForwarder : ObservableObject {
|
||||||
|
|
||||||
private let logger = Logger()
|
private let logger = Logger()
|
||||||
|
|
||||||
|
private let queue = OperationQueue()
|
||||||
|
|
||||||
|
private var condition = NSCondition()
|
||||||
|
private var latestText = ""
|
||||||
|
|
||||||
func connect(destination : String) {
|
func connect(destination : String) {
|
||||||
logger.debug("Attempting to connect to \(destination)")
|
connecting = true
|
||||||
client = TCPClient(address: destination, port: Int32(port))
|
queue.addOperation {
|
||||||
guard let client = client else { return }
|
self.logger.debug("Attempting to connect to \(destination)")
|
||||||
switch client.connect(timeout: 10) {
|
self.client = TCPClient(address: destination, port: Int32(self.port))
|
||||||
case .success:
|
guard let client = self.client else {
|
||||||
connected = true
|
OperationQueue.main.addOperation { self.connecting = false }
|
||||||
logger.debug("Connected to \(destination)")
|
return
|
||||||
case .failure(let error):
|
}
|
||||||
client.close()
|
switch client.connect(timeout: 10) {
|
||||||
self.client = nil
|
case .success:
|
||||||
logger.error("Failed to connect to \(destination): \(String(describing: error))")
|
OperationQueue.main.addOperation { self.connected = true }
|
||||||
break
|
self.logger.debug("Connected to \(destination)")
|
||||||
|
case .failure(let error):
|
||||||
|
client.close()
|
||||||
|
self.client = nil
|
||||||
|
self.logger.error("Failed to connect to \(destination): \(String(describing: error))")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
OperationQueue.main.addOperation { self.connecting = false }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -52,7 +67,13 @@ class SpeechForwarder : ObservableObject {
|
||||||
}
|
}
|
||||||
|
|
||||||
guard let client = client else { return }
|
guard let client = client else { return }
|
||||||
|
|
||||||
|
condition.lock()
|
||||||
client.close()
|
client.close()
|
||||||
|
self.client = nil
|
||||||
|
condition.broadcast()
|
||||||
|
condition.unlock()
|
||||||
|
|
||||||
connected = false
|
connected = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -64,25 +85,25 @@ class SpeechForwarder : ObservableObject {
|
||||||
// app’s interface, so process the results on the app’s
|
// app’s interface, so process the results on the app’s
|
||||||
// main queue.
|
// main queue.
|
||||||
OperationQueue.main.addOperation {
|
OperationQueue.main.addOperation {
|
||||||
switch authStatus {
|
switch authStatus {
|
||||||
case .authorized:
|
case .authorized:
|
||||||
break
|
break
|
||||||
|
|
||||||
case .denied:
|
case .denied:
|
||||||
self.listening = false
|
self.listening = false
|
||||||
break
|
break
|
||||||
|
|
||||||
case .restricted:
|
case .restricted:
|
||||||
self.listening = false
|
self.listening = false
|
||||||
break
|
break
|
||||||
|
|
||||||
case .notDetermined:
|
case .notDetermined:
|
||||||
self.listening = false
|
self.listening = false
|
||||||
break
|
break
|
||||||
|
|
||||||
default:
|
default:
|
||||||
self.listening = false
|
self.listening = false
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -111,16 +132,21 @@ class SpeechForwarder : ObservableObject {
|
||||||
|
|
||||||
if (!self.listening) {
|
if (!self.listening) {
|
||||||
logger.debug("Stopped listening")
|
logger.debug("Stopped listening")
|
||||||
audioEngine.stop()
|
|
||||||
recognitionRequest?.endAudio()
|
recognitionRequest?.endAudio()
|
||||||
|
audioEngine.stop()
|
||||||
|
audioEngine.inputNode.removeTap(onBus: 0)
|
||||||
recognitionTask?.cancel()
|
recognitionTask?.cancel()
|
||||||
audioEngine.inputNode.removeTap(onBus: 0);
|
|
||||||
audioEngine.inputNode.reset()
|
self.recognitionRequest = nil
|
||||||
|
self.recognitionTask = nil
|
||||||
|
condition.lock()
|
||||||
|
self.listening = false
|
||||||
|
condition.broadcast()
|
||||||
|
condition.unlock()
|
||||||
switch (client.send(data: isListening())) {
|
switch (client.send(data: isListening())) {
|
||||||
case .success:
|
case .success:
|
||||||
break
|
break
|
||||||
case .failure(let error):
|
case .failure(let error):
|
||||||
self.listening = false
|
|
||||||
logger.error("Failed to send header: \(String(describing: error))")
|
logger.error("Failed to send header: \(String(describing: error))")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -130,43 +156,109 @@ class SpeechForwarder : ObservableObject {
|
||||||
return pack("<hh", [LISTEN_STATE_MSG, listening ? 1 : 0])
|
return pack("<hh", [LISTEN_STATE_MSG, listening ? 1 : 0])
|
||||||
}
|
}
|
||||||
|
|
||||||
private func send(latestText : String) {
|
private func send() {
|
||||||
guard let client = client else { return }
|
var stringLastSent = ""
|
||||||
var commonChars = self.textHeard.count
|
var stringToSend = ""
|
||||||
|
var canSend = true
|
||||||
|
|
||||||
|
while true {
|
||||||
|
while (!canSend) {
|
||||||
|
logger.debug("Cannot send")
|
||||||
|
guard let client = client else {
|
||||||
|
logger.debug("Returning because client gone")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
guard let byteArray = client.read(2, timeout: 1) else {
|
||||||
|
logger.debug("Did not read data")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
let data = Data(byteArray)
|
||||||
|
do {
|
||||||
|
let unpacked = try unpack("<h", data)
|
||||||
|
canSend = (unpacked[0] as? Int == LISTEN_SEND_MORE)
|
||||||
|
logger.debug("Updated canSend")
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
logger.debug("Unpack failed")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.debug("Can send")
|
||||||
|
|
||||||
|
condition.lock()
|
||||||
|
while (stringLastSent == latestText) {
|
||||||
|
if (!self.listening) {
|
||||||
|
condition.unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
condition.wait()
|
||||||
|
if (!self.listening) {
|
||||||
|
condition.unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
guard client != nil else {
|
||||||
|
condition.unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stringToSend = latestText
|
||||||
|
condition.unlock()
|
||||||
|
|
||||||
|
if send(latestText: stringToSend, lastSent: stringLastSent) {
|
||||||
|
stringLastSent = stringToSend
|
||||||
|
canSend = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func send(latestText : String, lastSent: String) -> Bool {
|
||||||
|
guard let client = client else { return false }
|
||||||
|
var commonChars = lastSent.count
|
||||||
while (commonChars > 0) {
|
while (commonChars > 0) {
|
||||||
if (latestText.prefix(commonChars) == self.textHeard.prefix(commonChars)) {
|
if (latestText.prefix(commonChars) == lastSent.prefix(commonChars)) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
commonChars -= 1
|
commonChars -= 1
|
||||||
}
|
}
|
||||||
var stringToSend = ""
|
var stringToSend = ""
|
||||||
if (commonChars < self.textHeard.count) {
|
if (commonChars < lastSent.count) {
|
||||||
stringToSend = String(repeating: "\u{7f}", count: self.textHeard.count - commonChars)
|
stringToSend = String(repeating: "\u{7f}", count: lastSent.count - commonChars)
|
||||||
}
|
}
|
||||||
stringToSend.append(contentsOf: latestText.suffix(latestText.count - commonChars).replacingOccurrences(of: "\n", with: "\r"))
|
stringToSend.append(contentsOf: latestText.suffix(latestText.count - commonChars).replacingOccurrences(of: "\n", with: "\r"))
|
||||||
|
|
||||||
if (stringToSend.count > 0) {
|
if (stringToSend.count == 0) {
|
||||||
// TODO - Handle strings to send that are longer than 64K (doubt that would happen though)
|
return false
|
||||||
let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue))
|
}
|
||||||
let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding
|
|
||||||
if let bytes = stringToSend.data(using: encoding) {
|
// JSR_TODO - Handle strings to send that are longer than 64K (doubt that would happen though)
|
||||||
switch (client.send(data: pack("<hh", [LISTEN_TEXT_MSG, bytes.count]))) {
|
let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue))
|
||||||
case .success:
|
let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding
|
||||||
switch (client.send(data: bytes)) {
|
if let bytes = stringToSend.data(using: encoding) {
|
||||||
case .success:
|
switch (client.send(data: pack("<hh", [LISTEN_TEXT_MSG, bytes.count]))) {
|
||||||
self.textHeard = latestText
|
case .success:
|
||||||
logger.debug("Sent text \"\(stringToSend)\"")
|
switch (client.send(data: bytes)) {
|
||||||
break
|
case .success:
|
||||||
case .failure(let error):
|
logger.debug("Sent text \"\(stringToSend)\"")
|
||||||
self.listening = false
|
break
|
||||||
logger.error("Failed to send text: \(String(describing: error))")
|
case .failure(let error):
|
||||||
|
OperationQueue.main.addOperation {
|
||||||
|
if (self.listening) {
|
||||||
|
self.listen()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.error("Failed to send text: \(String(describing: error))")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case .failure(let error):
|
||||||
|
OperationQueue.main.addOperation {
|
||||||
|
if (self.listening) {
|
||||||
|
self.listen()
|
||||||
}
|
}
|
||||||
case .failure(let error):
|
}
|
||||||
self.listening = false
|
logger.error("Failed to send text: \(String(describing: error))")
|
||||||
logger.error("Failed to send text: \(String(describing: error))")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
private func startRecording() throws {
|
private func startRecording() throws {
|
||||||
|
@ -198,6 +290,13 @@ class SpeechForwarder : ObservableObject {
|
||||||
}
|
}
|
||||||
|
|
||||||
self.textHeard = ""
|
self.textHeard = ""
|
||||||
|
self.latestText = ""
|
||||||
|
self.sending = true
|
||||||
|
|
||||||
|
queue.addOperation {
|
||||||
|
self.send()
|
||||||
|
OperationQueue.main.addOperation { self.sending = false }
|
||||||
|
}
|
||||||
|
|
||||||
// Create a recognition task for the speech recognition session.
|
// Create a recognition task for the speech recognition session.
|
||||||
// Keep a reference to the task so that it can be canceled.
|
// Keep a reference to the task so that it can be canceled.
|
||||||
|
@ -206,9 +305,14 @@ class SpeechForwarder : ObservableObject {
|
||||||
|
|
||||||
if let result = result {
|
if let result = result {
|
||||||
// Update the text view with the results.
|
// Update the text view with the results.
|
||||||
self.send(latestText: result.bestTranscription.formattedString)
|
self.condition.lock()
|
||||||
|
self.latestText = result.bestTranscription.formattedString
|
||||||
|
self.condition.broadcast()
|
||||||
|
self.condition.unlock()
|
||||||
|
|
||||||
|
OperationQueue.main.addOperation { self.textHeard = result.bestTranscription.formattedString }
|
||||||
|
|
||||||
isFinal = result.isFinal
|
isFinal = result.isFinal
|
||||||
print("Text \(result.bestTranscription.formattedString)")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if error != nil {
|
if error != nil {
|
||||||
|
@ -216,20 +320,10 @@ class SpeechForwarder : ObservableObject {
|
||||||
}
|
}
|
||||||
|
|
||||||
if error != nil || isFinal {
|
if error != nil || isFinal {
|
||||||
// Stop recognizing speech if there is a problem.
|
OperationQueue.main.addOperation {
|
||||||
self.audioEngine.stop()
|
if (self.listening) {
|
||||||
inputNode.removeTap(onBus: 0)
|
self.listen()
|
||||||
|
}
|
||||||
self.recognitionRequest = nil
|
|
||||||
self.recognitionTask = nil
|
|
||||||
self.listening = false
|
|
||||||
self.logger.debug("Stopped listening")
|
|
||||||
guard let client = self.client else { return }
|
|
||||||
switch (client.send(data: self.isListening())) {
|
|
||||||
case .success:
|
|
||||||
break
|
|
||||||
case .failure(let error):
|
|
||||||
self.logger.error("Failed to send header: \(String(describing: error))")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user