From 923d0bf9674cae20e6787dc1051050807fee42f7 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Tue, 15 Mar 2022 23:58:04 -0400 Subject: [PATCH] A fairly big rework of the code to split the network handling code from the speech handling code. Also, introduce a thread for reading from the socket and a separate thread for writing to the socket. That way, disconnections made by the NDA are handled correctly. --- ListenerGS.xcodeproj/project.pbxproj | 4 + ListenerGS/GSConnection.swift | 346 +++++++++++++++++++++++++++ ListenerGS/GSView.swift | 59 +++-- ListenerGS/Info.plist | 2 +- ListenerGS/SpeechForwarder.swift | 281 +++------------------- ListenerGS/SwiftSocket/ytcpsocket.c | 1 + 6 files changed, 428 insertions(+), 265 deletions(-) create mode 100644 ListenerGS/GSConnection.swift diff --git a/ListenerGS.xcodeproj/project.pbxproj b/ListenerGS.xcodeproj/project.pbxproj index f676bd2..c7a0fd5 100644 --- a/ListenerGS.xcodeproj/project.pbxproj +++ b/ListenerGS.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 9D05BAAA27DFDE6300D9CC4B /* GSConnection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D05BAA927DFDE6300D9CC4B /* GSConnection.swift */; }; 9D5155F326A1EF7B0075EBC7 /* ListenerGSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D5155F226A1EF7B0075EBC7 /* ListenerGSApp.swift */; }; 9D5155F726A1EF7C0075EBC7 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 9D5155F626A1EF7C0075EBC7 /* Assets.xcassets */; }; 9D5155FA26A1EF7C0075EBC7 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 9D5155F926A1EF7C0075EBC7 /* Preview Assets.xcassets */; }; @@ -48,6 +49,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + 9D05BAA927DFDE6300D9CC4B /* GSConnection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GSConnection.swift; sourceTree = ""; }; 9D0DC15826F2E47A007EB92D /* ListenerGS.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = ListenerGS.entitlements; sourceTree = ""; }; 9D5155EF26A1EF7B0075EBC7 /* ListenerGS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = ListenerGS.app; sourceTree = BUILT_PRODUCTS_DIR; }; 9D5155F226A1EF7B0075EBC7 /* ListenerGSApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ListenerGSApp.swift; sourceTree = ""; }; @@ -141,6 +143,7 @@ 9DCCDACB271FB87100F311DF /* GSDestinations.swift */, 9DD8905F2772D3B20084A894 /* GSView.swift */, 9D6ED239271E6BD600D773CD /* SpeechForwarder.swift */, + 9D05BAA927DFDE6300D9CC4B /* GSConnection.swift */, 9DD8905E27726C140084A894 /* ListenerGS Icon.pxm */, 9D51566326A36F530075EBC7 /* BinUtils */, 9D51563626A36AD60075EBC7 /* SwiftSocket */, @@ -365,6 +368,7 @@ buildActionMask = 2147483647; files = ( 9D51565726A36B410075EBC7 /* TCPClient.swift in Sources */, + 9D05BAAA27DFDE6300D9CC4B /* GSConnection.swift in Sources */, 9D6F27092728EF410089585E /* MainView.swift in Sources */, 9D51565526A36B410075EBC7 /* UDPClient.swift in Sources */, 9DD67CF02728F5B700243FC6 /* DestinationsView.swift in Sources */, diff --git a/ListenerGS/GSConnection.swift b/ListenerGS/GSConnection.swift new file mode 100644 index 0000000..01398b2 --- /dev/null +++ b/ListenerGS/GSConnection.swift @@ -0,0 +1,346 @@ +// +// GSConnection.swift +// ListenerGS +// +// Created by Jeremy Rand on 2022-03-14. +// + +import Foundation +import os + +struct GSConnectionErrorMessage: Identifiable { + var id: String { message } + let title: String + let message: String +} + +enum GSConnectionState { + case disconnected + case connecting + case connected + case listening + case stoplistening +} + +extension GSConnectionState: CustomStringConvertible +{ + var description: String { + switch self { + case .disconnected: + return "disconnected" + case .connecting: + return "connecting" + case .connected: + return "connected" + case .listening: + return "listening" + case .stoplistening: + return "stop listening" + } + } +} + +protocol SpeechForwarderProtocol +{ + func startListening() -> Bool + func stopListening() +} + +class GSConnection : ObservableObject { + @Published var state = GSConnectionState.disconnected + @Published var textHeard = "" + @Published var errorMessage : GSConnectionErrorMessage? + + var speechForwarder : SpeechForwarderProtocol? + + let LISTEN_STATE_MSG = 1 + let LISTEN_TEXT_MSG = 2 + let LISTEN_SEND_MORE = 3 + + let port = 19026 + private var destination = "" + private var client: TCPClient? + + private let logger = Logger() + + private let readQueue = OperationQueue() + private let writeQueue = OperationQueue() + + private var condition = NSCondition() + private var stopListeningFlag = false + private var canSend = true + + func changeState(newState : GSConnectionState) + { + if (state == newState) { + return; + } + + var legalTransition = false + switch (newState) + { + case .disconnected: + legalTransition = ((state == .connected) || (state == .connecting)) + + case .connecting: + legalTransition = (state == .disconnected) + + case .connected: + legalTransition = ((state == .connecting) || (state == .listening) || (state == .stoplistening)) + + case .listening: + legalTransition = (state == .connected) + + case .stoplistening: + legalTransition = ((state == .connected) || (state == .listening)) + } + + if (!legalTransition) { + logger.error("Illegal requested state transition from \(self.state) to \(newState)") + errorOccurred(title: "Bad State Change", message: "Illegal state transition from \(self.state) to \(newState)") + } else { + state = newState + } + } + + func errorOccurred(title: String, message : String) + { + OperationQueue.main.addOperation { + self.errorMessage = GSConnectionErrorMessage(title: title, message: message) + } + } + + private func connectionFailed() { + errorOccurred(title: "Connect Error", message: "Failed to connect to \(destination)") + changeState(newState:.disconnected) + } + + private func connectionSuccessful() + { + changeState(newState:.connected) + logger.debug("Connected to \(self.destination)") + } + + private func doConnect() { + logger.debug("Attempting to connect to \(self.destination)") + client = TCPClient(address: destination, port: Int32(port)) + guard let client = client else { + OperationQueue.main.addOperation { self.connectionFailed() } + return + } + switch client.connect(timeout: 10) { + case .success: + OperationQueue.main.addOperation { self.connectionSuccessful() } + case .failure(let error): + client.close() + self.client = nil + logger.error("Failed to connect to \(self.destination): \(String(describing: error))") + OperationQueue.main.addOperation { self.connectionFailed() } + return + } + + while (true) { + guard let byteArray = client.read(2) else { + break + } + let data = Data(byteArray) + do { + let unpacked = try unpack(" Bool { + guard let client = client else { return false } + + switch (client.send(data: pack(" Bool { + guard let client = client else { return false } + var commonChars = lastSent.count + while (commonChars > 0) { + if (latestText.prefix(commonChars) == lastSent.prefix(commonChars)) { + break + } + commonChars -= 1 + } + var stringToSend = "" + if (commonChars < lastSent.count) { + stringToSend = String(repeating: "\u{7f}", count: lastSent.count - commonChars) + } + stringToSend.append(contentsOf: latestText.suffix(latestText.count - commonChars).replacingOccurrences(of: "\n", with: "\r")) + + if (stringToSend.count == 0) { + return false + } + + // JSR_TODO - Handle strings to send that are longer than 64K (doubt that would happen though) + let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue)) + let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding + if let bytes = stringToSend.data(using: encoding) { + switch (client.send(data: pack("CFBundleShortVersionString 1.0 CFBundleVersion - 445 + 483 LSApplicationCategoryType public.app-category.utilities LSRequiresIPhoneOS diff --git a/ListenerGS/SpeechForwarder.swift b/ListenerGS/SpeechForwarder.swift index 5b504c4..c4d5337 100644 --- a/ListenerGS/SpeechForwarder.swift +++ b/ListenerGS/SpeechForwarder.swift @@ -9,19 +9,9 @@ import Foundation import os import Speech -class SpeechForwarder : ObservableObject { - @Published var listening = false - @Published var connected = false - @Published var connecting = false - @Published var textHeard = "" - @Published var sending = false +class SpeechForwarder : SpeechForwarderProtocol { - let LISTEN_STATE_MSG = 1 - let LISTEN_TEXT_MSG = 2 - let LISTEN_SEND_MORE = 3 - - let port = 19026 - private var client: TCPClient? + private var connection : GSConnection private let speechRecognizer = SFSpeechRecognizer(locale: Locale(identifier: Locale.preferredLanguages[0]))! @@ -33,234 +23,47 @@ class SpeechForwarder : ObservableObject { private let logger = Logger() - private let queue = OperationQueue() - - private var condition = NSCondition() - private var latestText = "" - - func connect(destination : String) { - connecting = true - queue.addOperation { - self.logger.debug("Attempting to connect to \(destination)") - self.client = TCPClient(address: destination, port: Int32(self.port)) - guard let client = self.client else { - OperationQueue.main.addOperation { self.connecting = false } - return - } - switch client.connect(timeout: 10) { - case .success: - OperationQueue.main.addOperation { self.connected = true } - self.logger.debug("Connected to \(destination)") - case .failure(let error): - client.close() - self.client = nil - self.logger.error("Failed to connect to \(destination): \(String(describing: error))") - break - } - OperationQueue.main.addOperation { self.connecting = false } - } + init(connection : GSConnection) { + self.connection = connection } - func disconnect() { - if (listening) { - listen() - } - - guard let client = client else { return } - - condition.lock() - client.close() - self.client = nil - condition.broadcast() - condition.unlock() - - connected = false - } - - func listen() { - self.listening.toggle() - if (self.listening) { - SFSpeechRecognizer.requestAuthorization { authStatus in - // The authorization status results in changes to the - // app’s interface, so process the results on the app’s - // main queue. - OperationQueue.main.addOperation { - switch authStatus { - case .authorized: - break - - case .denied: - self.listening = false - break - - case .restricted: - self.listening = false - break - - case .notDetermined: - self.listening = false - break - - default: - self.listening = false - break - } - } - } - } - - guard let client = client else { return } - if (self.listening) { - switch (client.send(data: isListening())) { - case .success: + func startListening() -> Bool { + SFSpeechRecognizer.requestAuthorization { authStatus in + OperationQueue.main.addOperation { + switch authStatus { + case .authorized: break - case .failure(let error): - self.listening = false - logger.error("Unable to send header: \(String(describing: error))") + + case .denied, .restricted, .notDetermined: + self.connection.stopListening() + + default: + self.connection.stopListening() + } } } - - if (self.listening) { - do { - try startRecording() - logger.debug("Started listening") - } - catch { - self.listening = false - } - } - - if (!self.listening) { - logger.debug("Stopped listening") - recognitionRequest?.endAudio() - audioEngine.stop() - audioEngine.inputNode.removeTap(onBus: 0) - recognitionTask?.cancel() - - self.recognitionRequest = nil - self.recognitionTask = nil - condition.lock() - self.listening = false - condition.broadcast() - condition.unlock() - switch (client.send(data: isListening())) { - case .success: - break - case .failure(let error): - logger.error("Failed to send header: \(String(describing: error))") - } - } - } - - private func isListening() -> Data { - return pack(" Bool { - guard let client = client else { return false } - var commonChars = lastSent.count - while (commonChars > 0) { - if (latestText.prefix(commonChars) == lastSent.prefix(commonChars)) { - break - } - commonChars -= 1 - } - var stringToSend = "" - if (commonChars < lastSent.count) { - stringToSend = String(repeating: "\u{7f}", count: lastSent.count - commonChars) - } - stringToSend.append(contentsOf: latestText.suffix(latestText.count - commonChars).replacingOccurrences(of: "\n", with: "\r")) - - if (stringToSend.count == 0) { + catch { return false } - - // JSR_TODO - Handle strings to send that are longer than 64K (doubt that would happen though) - let nsEnc = CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(CFStringBuiltInEncodings.macRoman.rawValue)) - let encoding = String.Encoding(rawValue: nsEnc) // String.Encoding - if let bytes = stringToSend.data(using: encoding) { - switch (client.send(data: pack("