Apple-2-Tools/ListenerApp
1
0
Fork 0
mirror of https://github.com/jeremysrand/ListenerApp.git synced 2024-12-21 13:29:53 +00:00
Code Issues Projects Releases Wiki Activity

Add code to pace the text to the GS to reduce the amount of re-typing required.

Browse Source
This commit is contained in:
Jeremy Rand 2022-02-22 23:52:00 -05:00
parent 241190600e
commit b2c2cbd587
3 changed files with 171 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ListenerGS/GSView.swift
View File

@ -52,7 +52,7 @@ struct GSView: View {
speechForwarder.connect(destination: ipAddress)
}
}
.disabled(false)
.disabled(speechForwarder.connecting)
.buttonStyle(GSButtonStyle())
Button(speechForwarder.listening ?
@ -60,7 +60,7 @@ struct GSView: View {
"\(Image(systemName: "ear.and.waveform")) Listen and Send Text") {
speechForwarder.listen()
}
.disabled(!speechForwarder.connected)
.disabled((!speechForwarder.connected) || (!speechForwarder.listening && speechForwarder.sending))
.buttonStyle(GSButtonStyle())
}
.fixedSize(horizontal: true, vertical: false)

2
ListenerGS/Info.plist
View File

@ -19,7 +19,7 @@
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>390</string>
<string>434</string>
<key>LSApplicationCategoryType</key>
<string>public.app-category.utilities</string>
<key>LSRequiresIPhoneOS</key>

242
ListenerGS/SpeechForwarder.swift
View File

@ -12,10 +12,13 @@ import Speech
class SpeechForwarder : ObservableObject {
@Published var listening = false
@Published var connected = false
@Published var connecting = false
@Published var textHeard = ""
@Published var sending = false
let LISTEN_STATE_MSG = 1
let LISTEN_TEXT_MSG = 2
let LISTEN_SEND_MORE = 3
let port = 19026
private var client: TCPClient?
@ -30,19 +33,31 @@ class SpeechForwarder : ObservableObject {
private let logger = Logger()
private let queue = OperationQueue()
private var condition = NSCondition()
private var latestText = ""
func connect(destination : String) {
logger.debug("Attempting to connect to \(destination)")
client = TCPClient(address: destination, port: Int32(port))
guard let client = client else { return }
switch client.connect(timeout: 10) {
case .success:
connected = true
logger.debug("Connected to \(destination)")
case .failure(let error):
client.close()
self.client = nil
logger.error("Failed to connect to \(destination): \(String(describing: error))")
break
connecting = true
queue.addOperation {
self.logger.debug("Attempting to connect to \(destination)")
self.client = TCPClient(address: destination, port: Int32(self.port))
guard let client = self.client else {
OperationQueue.main.addOperation { self.connecting = false }
return
}
switch client.connect(timeout: 10) {
case .success:
OperationQueue.main.addOperation { self.connected = true }
self.logger.debug("Connected to \(destination)")
case .failure(let error):
client.close()
self.client = nil
self.logger.error("Failed to connect to \(destination): \(String(describing: error))")
break
}
OperationQueue.main.addOperation { self.connecting = false }
}
}
@ -52,7 +67,13 @@ class SpeechForwarder : ObservableObject {
}
guard let client = client else { return }
condition.lock()
client.close()
self.client = nil
condition.broadcast()
condition.unlock()
connected = false
}
@ -64,25 +85,25 @@ class SpeechForwarder : ObservableObject {
// app’s interface, so process the results on the app’s
// main queue.
OperationQueue.main.addOperation {
switch authStatus {
case .authorized:
break
case .denied:
self.listening = false
break
switch authStatus {
case .authorized:
break
case .denied:
self.listening = false
break
case .restricted:
self.listening = false
break
case .restricted:
self.listening = false
break
case .notDetermined:
self.listening = false
break
default:
self.listening = false
break
case .notDetermined:
self.listening = false
break
default:
self.listening = false
break
}
}
}
@ -111,16 +132,21 @@ class SpeechForwarder : ObservableObject {
if (!self.listening) {
logger.debug("Stopped listening")
audioEngine.stop()
recognitionRequest?.endAudio()
audioEngine.stop()
audioEngine.inputNode.removeTap(onBus: 0)
recognitionTask?.cancel()
audioEngine.inputNode.removeTap(onBus: 0);
audioEngine.inputNode.reset()
self.recognitionRequest = nil
self.recognitionTask = nil
condition.lock()
self.listening = false
condition.broadcast()
condition.unlock()
switch (client.send(data: isListening())) {
case .success:
break
case .failure(let error):
self.listening = false
logger.error("Failed to send header: \(String(describing: error))")
}
}
@ -130,43 +156,109 @@ class SpeechForwarder : ObservableObject {
return pack("<hh", [LISTEN_STATE_MSG, listening ? 1 : 0])
}
private func send(latestText : String) {
guard let client = client else { return }
var commonChars = self.textHeard.count
private func send() {
var stringLastSent = ""
var stringToSend = ""
var canSend = true
while true {
while (!canSend) {
logger.debug("Cannot send")
guard let client = client else {
logger.debug("Returning because client gone")
return
}
guard let byteArray = client.read(2, timeout: 1) else {
logger.debug("Did not read data")
continue
}
let data = Data(byteArray)
do {
let unpacked = try unpack("<h", data)
canSend = (unpacked[0] as? Int == LISTEN_SEND_MORE)
logger.debug("Updated canSend")
}
catch {
logger.debug("Unpack failed")
continue
}
}
logger.debug("Can send")
condition.lock()
while (stringLastSent == latestText) {
if (!self.listening) {
condition.unlock()
return
}
condition.wait()
if (!self.listening) {
condition.unlock()
return
}
guard client != nil else {
condition.unlock()
return
}
}
stringToSend = latestText
condition.unlock()
if send(latestText: stringToSend, lastSent: stringLastSent) {
stringLastSent = stringToSend
canSend = false
}
}
}
private func send(latestText : String, lastSent: String) -> Bool {
guard let client = client else { return false }
var commonChars = lastSent.count
while (commonChars > 0) {
if (latestText.prefix(commonChars) == self.textHeard.prefix(commonChars)) {
if (latestText.prefix(commonChars) == lastSent.prefix(commonChars)) {
break
}
commonChars -= 1
}
var stringToSend = ""
if (commonChars < self.textHeard.count) {
stringToSend = String(repeating: "\u{7f}", count: self.textHeard.count - commonChars)
if (commonChars < lastSent.count) {
stringToSend = String(repeating: "\u{7f}", count: lastSent.count - commonChars)
}
stringToSend.append(contentsOf: latestText.suffix(latestText.count - commonChars).replacingOccurrences(of: "\n", with: "\r"))
if (stringToSend.count > 0) {
// TODO - Handle strings to send that are longer than 64K (doubt that would happen though)
let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue))
let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding
if let bytes = stringToSend.data(using: encoding) {
switch (client.send(data: pack("<hh", [LISTEN_TEXT_MSG, bytes.count]))) {
case .success:
switch (client.send(data: bytes)) {
case .success:
self.textHeard = latestText
logger.debug("Sent text \"\(stringToSend)\"")
break
case .failure(let error):
self.listening = false
logger.error("Failed to send text: \(String(describing: error))")
if (stringToSend.count == 0) {
return false
}
// JSR_TODO - Handle strings to send that are longer than 64K (doubt that would happen though)
let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue))
let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding
if let bytes = stringToSend.data(using: encoding) {
switch (client.send(data: pack("<hh", [LISTEN_TEXT_MSG, bytes.count]))) {
case .success:
switch (client.send(data: bytes)) {
case .success:
logger.debug("Sent text \"\(stringToSend)\"")
break
case .failure(let error):
OperationQueue.main.addOperation {
if (self.listening) {
self.listen()
}
}
logger.error("Failed to send text: \(String(describing: error))")
return false
}
case .failure(let error):
OperationQueue.main.addOperation {
if (self.listening) {
self.listen()
}
case .failure(let error):
self.listening = false
logger.error("Failed to send text: \(String(describing: error))")
}
}
logger.error("Failed to send text: \(String(describing: error))")
}
}
return true
}
private func startRecording() throws {
@ -198,6 +290,13 @@ class SpeechForwarder : ObservableObject {
}
self.textHeard = ""
self.latestText = ""
self.sending = true
queue.addOperation {
self.send()
OperationQueue.main.addOperation { self.sending = false }
}
// Create a recognition task for the speech recognition session.
// Keep a reference to the task so that it can be canceled.
@ -206,9 +305,14 @@ class SpeechForwarder : ObservableObject {
if let result = result {
// Update the text view with the results.
self.send(latestText: result.bestTranscription.formattedString)
self.condition.lock()
self.latestText = result.bestTranscription.formattedString
self.condition.broadcast()
self.condition.unlock()
OperationQueue.main.addOperation { self.textHeard = result.bestTranscription.formattedString }
isFinal = result.isFinal
print("Text \(result.bestTranscription.formattedString)")
}
if error != nil {
@ -216,20 +320,10 @@ class SpeechForwarder : ObservableObject {
}
if error != nil || isFinal {
// Stop recognizing speech if there is a problem.
self.audioEngine.stop()
inputNode.removeTap(onBus: 0)
self.recognitionRequest = nil
self.recognitionTask = nil
self.listening = false
self.logger.debug("Stopped listening")
guard let client = self.client else { return }
switch (client.send(data: self.isListening())) {
case .success:
break
case .failure(let error):
self.logger.error("Failed to send header: \(String(describing: error))")
OperationQueue.main.addOperation {
if (self.listening) {
self.listen()
}
}
}
}