diff --git a/ios/Classes/Camera/AudioSession.swift b/ios/Classes/Camera/AudioSession.swift new file mode 100644 index 0000000..d6c7746 --- /dev/null +++ b/ios/Classes/Camera/AudioSession.swift @@ -0,0 +1,71 @@ +import AVFoundation + +/// Idempotent helpers for upgrading the app's shared `AVAudioSession` +/// to the union of categories needed for capture without trampling +/// what other modules (audio player, video_player) had set. Pattern +/// mirrors `camera_avfoundation`'s `upgradeAudioSessionCategory` — +/// only ever WIDENS the category, never narrows. +/// +/// We pair this with the session's +/// `automaticallyConfiguresApplicationAudioSession = false` + +/// `usesApplicationAudioSession = true` (set in [CameraSession.init]) +/// so AVCaptureSession doesn't yank the category back. +enum AudioSession { + /// Upgrade the shared category to include `.playAndRecord` (and + /// the given options union'd with whatever's already set). No-op + /// when the union equals the current state, so this is cheap to + /// call on every recording start. + static func upgradeForRecording() { + upgrade( + requestedCategory: .playAndRecord, + options: [.defaultToSpeaker, .allowBluetoothA2DP, .allowAirPlay] + ) + } + + private static func upgrade( + requestedCategory: AVAudioSession.Category, + options: AVAudioSession.CategoryOptions + ) { + let playCategories: Set = [.playback, .playAndRecord] + let recordCategories: Set = [.record, .playAndRecord] + let currentCategory = AVAudioSession.sharedInstance().category + let requiredCategories: Set = [ + requestedCategory, currentCategory, + ] + + let requiresPlay = !requiredCategories.isDisjoint(with: playCategories) + let requiresRecord = !requiredCategories.isDisjoint(with: recordCategories) + + var finalCategory = requestedCategory + if requiresPlay && requiresRecord { + finalCategory = .playAndRecord + } else if requiresPlay { + finalCategory = .playback + } else if requiresRecord { + finalCategory = .record + } + + let finalOptions = AVAudioSession.sharedInstance().categoryOptions + .union(options) + + if finalCategory == currentCategory + && finalOptions == AVAudioSession.sharedInstance().categoryOptions + { + return + } + + try? AVAudioSession.sharedInstance().setCategory( + finalCategory, + options: finalOptions + ) + // With AVCaptureSession.usesApplicationAudioSession = true and + // automaticallyConfiguresApplicationAudioSession = false, the + // app owns activation — without this, the mic input never + // delivers sample buffers. Telegram does the same from + // ManagedAudioSession.activate (setActive(true)). + try? AVAudioSession.sharedInstance().setActive( + true, + options: [.notifyOthersOnDeactivation] + ) + } +} diff --git a/ios/Classes/Camera/CameraInstance.swift b/ios/Classes/Camera/CameraInstance.swift new file mode 100644 index 0000000..673e852 --- /dev/null +++ b/ios/Classes/Camera/CameraInstance.swift @@ -0,0 +1,506 @@ +import AVFoundation +import Flutter +import Foundation + +/// One per `UxCameraController` on the Dart side. Owns its +/// `AVCaptureSession`, the texture-backed preview pipeline, photo +/// output, audio + video data outputs, and the +/// [VideoRecorder] when a recording is in flight. Multiple instances +/// coexist; the `[CameraPlugin]` keys them by `handle`. +/// +/// **Threading**: +/// - Session config (add/remove inputs/outputs, start, stop) runs on +/// `sessionQueue` (serial). +/// - Video sample-buffer delegate fires on `videoBufferQueue` (serial). +/// - Audio sample-buffer delegate fires on `audioBufferQueue` (serial). +/// - [VideoRecorder] mutates its writer state on `recorderQueue` (serial). +/// - All public completions land on `.main`. +final class CameraInstance { + let handle: Int + + /// Called whenever the instance has an event to forward to Dart. + /// The payload is the `{event, handle, …}` map the EventChannel + /// emits. Set by the plugin at construction. + var onEvent: (([String: Any]) -> Void)? + + private let session = CameraSession() + private let sink = PreviewSink() + private let photoOutput = PhotoOutput() + private let orientation = DeviceOrientationBridge() + + private let sessionQueue: DispatchQueue + private let videoBufferQueue: DispatchQueue + private let audioBufferQueue: DispatchQueue + private let recorderQueue: DispatchQueue + + private var device: AVCaptureDevice? + private var deviceInput: AVCaptureDeviceInput? + private var videoDataOutput: AVCaptureVideoDataOutput? + + private var audioDevice: AVCaptureDevice? + private var audioDeviceInput: AVCaptureDeviceInput? + private var audioDataOutput: AVCaptureAudioDataOutput? + + private let fanout: SampleFanout + + private var flashMode: AVCaptureDevice.FlashMode = .off + private var lockedOrientation: DeviceOrientationFlutter? + private var enableAudio: Bool = false + private var disposed = false + + /// Set during [startVideoRecording], cleared after the stop + /// completion fires. Lives on `sessionQueue` (set/cleared); the + /// [SampleFanout] holds a parallel reference (under its own lock) + /// for the videoBufferQueue / audioBufferQueue to access without + /// crossing queue boundaries per frame. + private var videoRecorder: VideoRecorder? + + /// `uniqueID` of the AVCaptureDevice this instance is currently + /// bound to, or `nil` after [dispose]. Used by [CameraPlugin] to + /// release its device claim. Read on `.main`. + var currentCameraId: String? { device?.uniqueID } + + /// Whether this instance was constructed with `enableAudio: true` + /// and therefore owns the app-global audio claim. Read on `.main`. + var audioClaimed: Bool { enableAudio } + + /// Hop to this instance's serial session queue. Public entry + /// point so the plugin can dispatch session work without + /// exposing the queue directly. + func sessionQueueAsync(_ block: @escaping () -> Void) { + sessionQueue.async(execute: block) + } + + /// Active-format dimensions in the camera sensor's natural + /// orientation (typically landscape — `1920×1080` etc.). Set + /// after `create` / `setDescription` configures the session. + private(set) var previewSize: CGSize = .zero + + /// Texture id handed back to Dart. Stable for the lifetime of + /// the instance. + private(set) var textureId: Int64 = -1 + + init(handle: Int) { + self.handle = handle + sessionQueue = DispatchQueue(label: "ux.camera.session.\(handle)") + videoBufferQueue = DispatchQueue(label: "ux.camera.video.\(handle)") + audioBufferQueue = DispatchQueue(label: "ux.camera.audio.\(handle)") + recorderQueue = DispatchQueue(label: "ux.camera.recorder.\(handle)") + fanout = SampleFanout(sink: sink) + + session.onRuntimeError = { [weak self] error in + self?.emit([ + "event": "sessionError", + "code": "session_runtime_error", + "description": error.localizedDescription, + ]) + } + session.onInterrupted = { [weak self] reason in + self?.emit(["event": "sessionInterrupted", "reason": reason]) + } + session.onResumed = { [weak self] in + self?.emit(["event": "sessionResumed"]) + } + } + + // MARK: - Lifecycle + + /// Synchronously configure the session for [cameraId]. Registers + /// the texture, attaches audio if requested, upgrades the audio + /// session, and starts the orientation bridge. Must run on + /// sessionQueue. + func create( + cameraId: String, + enableAudio: Bool, + registry: FlutterTextureRegistry + ) throws { + precondition(!disposed) + self.enableAudio = enableAudio + + textureId = sink.register(with: registry) + + if enableAudio { + // Widen the shared audio session category before we + // attach the mic input — matches `camera_avfoundation`'s + // defensive pattern. No-op if already widened. + AudioSession.upgradeForRecording() + } + + try configureSession(forDeviceUniqueID: cameraId, replacing: false) + + orientation.start { [weak self] next in + guard let self = self else { return } + self.sessionQueue.async { self.applyOrientationFollowDevice(next) } + self.emit([ + "event": "deviceOrientationChanged", + "orientation": next.rawValue, + ]) + } + } + + /// Start the session. Must run on sessionQueue. + func initialize() { + precondition(!disposed) + session.start() + } + + /// Tear everything down. Idempotent. Hard-cancels any in-flight + /// recording (drops queued audio, `cancelWriting`, deletes the + /// partial file — telegram-ios's `cancelRecording` path). Must run + /// on sessionQueue. + func dispose() { + if disposed { return } + disposed = true + + if let recorder = videoRecorder { + recorder.cancel() + videoRecorder = nil + fanout.recorder = nil + } + + session.stop() + if let input = deviceInput { session.av.removeInput(input) } + if let input = audioDeviceInput { session.av.removeInput(input) } + if let output = videoDataOutput { session.av.removeOutput(output) } + if let output = audioDataOutput { session.av.removeOutput(output) } + session.av.removeOutput(photoOutput.avOutput) + + videoDataOutput = nil + deviceInput = nil + device = nil + audioDeviceInput = nil + audioDataOutput = nil + audioDevice = nil + + orientation.stop() + sink.unregister() + onEvent = nil + } + + // MARK: - Camera flip + + /// Replace the video input device (audio stays attached). Returns + /// the new previewSize. Must run on sessionQueue. + func setDescription(cameraId: String) throws -> CGSize { + precondition(!disposed) + try configureSession(forDeviceUniqueID: cameraId, replacing: true) + return previewSize + } + + // MARK: - Flash + orientation + + func setFlashMode(_ mode: AVCaptureDevice.FlashMode) { + flashMode = mode + } + + func lockCaptureOrientation(_ next: DeviceOrientationFlutter) { + lockedOrientation = next + applyVideoOrientationOnPreview(next) + } + + func unlockCaptureOrientation() { + lockedOrientation = nil + applyOrientationFollowDevice(orientation.current) + } + + // MARK: - Photo + + func takePicture( + snapshot: DeviceOrientationFlutter, + completion: @escaping (Result) -> Void + ) { + photoOutput.take(orientation: snapshot, flashMode: flashMode, completion: completion) + } + + // MARK: - Video recording + + /// Begin a recording. Must run on sessionQueue. Throws on writer + /// setup failure (typically a path / file-system issue). + /// [snapshot] is the orientation embedded as the file's track + /// transform — when the user is holding the device landscape, + /// pass landscape here and the file plays back landscape. + func startVideoRecording( + snapshot: DeviceOrientationFlutter + ) throws { + precondition(!disposed) + guard videoRecorder == nil else { + throw NSError( + domain: "ux.camera", + code: -20, + userInfo: [ + NSLocalizedDescriptionKey: + "Recording already in flight" + ] + ) + } + guard let videoOutput = videoDataOutput else { + throw NSError( + domain: "ux.camera", + code: -21, + userInfo: [NSLocalizedDescriptionKey: "Video output unavailable"] + ) + } + + let url = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("ux_camera_\(UUID().uuidString).mp4") + + // Audio is viable only when both the device input attached AND + // the audio output can recommend writer settings. Empty + // recommended settings means the audio path can't be muxed — + // telegram-ios fails the whole recording in that case + // (CameraOutput.swift:397-401); we silent-fall back to + // video-only so an audio-permission glitch doesn't break the + // page. + var audioViable = enableAudio + && audioDeviceInput != nil + && audioDataOutput != nil + + let baseVideoSettings = videoOutput.recommendedVideoSettingsForAssetWriter( + writingTo: .mp4 + ) as? [String: Any] + + var baseAudioSettings: [String: Any] = [:] + if audioViable, let ao = audioDataOutput { + baseAudioSettings = (ao.recommendedAudioSettingsForAssetWriter( + writingTo: .mp4 + ) as? [String: Any]) ?? [:] + if baseAudioSettings.isEmpty { + audioViable = false + } + } + + let recorder = VideoRecorder( + url: url, + orientation: snapshot, + hasAudio: audioViable, + baseVideoSettings: baseVideoSettings, + baseAudioSettings: baseAudioSettings, + recorderQueue: recorderQueue + ) + recorder.onDiagnostic = { [weak self] msg in + self?.emit(["event": "diagnostic", "message": msg]) + } + try recorder.start() + + videoRecorder = recorder + // Publish the recorder under the fanout's lock so the buffer + // queues see it on their next sample. + fanout.recorder = recorder + } + + /// Stop the in-flight recording. Completion fires on + /// `recorderQueue` (which is `.async`'d here back to `.main` by + /// the plugin). Returns the file path or an error. + /// + /// The fanout reference stays attached until `finishWriting` + /// completes — the recorder relies on *post-stop* sample buffers + /// crossing `recordingStopSampleTime` to trigger `maybeFinish`. + /// Detaching the feed at the wrong moment (before stop) is what + /// caused the 3-second watchdog to be the only thing finishing + /// the writer. + func stopVideoRecording( + completion: @escaping (Result) -> Void + ) { + guard let recorder = videoRecorder else { + completion(.failure(NSError( + domain: "ux.camera", + code: -22, + userInfo: [NSLocalizedDescriptionKey: "No recording in flight"] + ))) + return + } + + recorder.stop { [weak self] outcome in + guard let self = self else { + completion(outcome) + return + } + self.sessionQueue.async { + self.fanout.recorder = nil + self.videoRecorder = nil + } + completion(outcome) + } + } + + // MARK: - Private + + private func configureSession( + forDeviceUniqueID cameraId: String, + replacing: Bool + ) throws { + guard let device = AVCaptureDevice(uniqueID: cameraId) else { + throw NSError( + domain: "ux.camera", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Camera \(cameraId) not found"] + ) + } + + var caughtError: NSError? + + session.configure { + if replacing, let oldInput = deviceInput { + session.av.removeInput(oldInput) + } + + do { + let newInput = try AVCaptureDeviceInput(device: device) + guard session.av.canAddInput(newInput) else { + throw NSError( + domain: "ux.camera", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Cannot add input"] + ) + } + session.av.addInput(newInput) + deviceInput = newInput + } catch let error as NSError { + caughtError = error + return + } + + if !replacing { + if session.av.canSetSessionPreset(.high) { + session.av.sessionPreset = .high + } + + let videoOutput = AVCaptureVideoDataOutput() + videoOutput.videoSettings = [ + kCVPixelBufferPixelFormatTypeKey as String: + kCVPixelFormatType_32BGRA, + ] + videoOutput.alwaysDiscardsLateVideoFrames = true + videoOutput.setSampleBufferDelegate( + fanout, + queue: videoBufferQueue + ) + if session.av.canAddOutput(videoOutput) { + session.av.addOutput(videoOutput) + } + videoDataOutput = videoOutput + + if session.av.canAddOutput(photoOutput.avOutput) { + session.av.addOutput(photoOutput.avOutput) + } + + if enableAudio, + let mic = AVCaptureDevice.default(for: .audio) { + do { + let audioInput = try AVCaptureDeviceInput(device: mic) + if session.av.canAddInput(audioInput) { + session.av.addInput(audioInput) + audioDevice = mic + audioDeviceInput = audioInput + } + } catch { + // Don't fail the whole setup over audio — fall + // through; the recording will simply have no + // audio track. + } + + let audioOutput = AVCaptureAudioDataOutput() + audioOutput.setSampleBufferDelegate( + fanout, + queue: audioBufferQueue + ) + if session.av.canAddOutput(audioOutput) { + session.av.addOutput(audioOutput) + audioDataOutput = audioOutput + } + } + } + + // Apply preview-output settings on the (new) connection. + if let videoConn = videoDataOutput?.connection(with: .video) { + if videoConn.isVideoOrientationSupported { + videoConn.videoOrientation = lockedOrientation?.avVideoOrientation + ?? orientation.current.avVideoOrientation + } + if videoConn.isVideoMirroringSupported { + videoConn.automaticallyAdjustsVideoMirroring = false + videoConn.isVideoMirrored = (device.position == .front) + } + } + + self.device = device + CaptureDevice.applyDefaults(device) + + let dims = CMVideoFormatDescriptionGetDimensions( + device.activeFormat.formatDescription + ) + previewSize = CGSize(width: CGFloat(dims.width), height: CGFloat(dims.height)) + } + + if let error = caughtError { + throw error + } + } + + private func applyOrientationFollowDevice(_ next: DeviceOrientationFlutter) { + // When a lock is in effect the preview ignores physical + // rotation — the lock wins. + guard lockedOrientation == nil else { return } + applyVideoOrientationOnPreview(next) + } + + private func applyVideoOrientationOnPreview(_ next: DeviceOrientationFlutter) { + guard let conn = videoDataOutput?.connection(with: .video), + conn.isVideoOrientationSupported else { + return + } + conn.videoOrientation = next.avVideoOrientation + } + + private func emit(_ extras: [String: Any]) { + var payload: [String: Any] = ["handle": handle] + payload.merge(extras, uniquingKeysWith: { _, new in new }) + DispatchQueue.main.async { [weak self] in self?.onEvent?(payload) } + } +} + +/// Single sample-buffer delegate for both video + audio outputs. +/// Forwards video frames to [PreviewSink] and (when a recording is +/// active) both video and audio sample buffers to [VideoRecorder]. +/// +/// The `recorder` reference is cross-queue: written from +/// `sessionQueue` (set on startVideoRecording, cleared on +/// stopVideoRecording), read from `videoBufferQueue` and +/// `audioBufferQueue` (once per sample). An `NSLock` guards each +/// access — cheap, ~tens of nanoseconds per frame. +private final class SampleFanout: NSObject, + AVCaptureVideoDataOutputSampleBufferDelegate, + AVCaptureAudioDataOutputSampleBufferDelegate +{ + private let sink: PreviewSink + private let recorderLock = NSLock() + private var _recorder: VideoRecorder? + + var recorder: VideoRecorder? { + get { + recorderLock.lock(); defer { recorderLock.unlock() } + return _recorder + } + set { + recorderLock.lock(); defer { recorderLock.unlock() } + _recorder = newValue + } + } + + init(sink: PreviewSink) { + self.sink = sink + } + + func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + if output is AVCaptureVideoDataOutput { + sink.receive(sampleBuffer: sampleBuffer) + recorder?.appendVideo(sampleBuffer) + } else if output is AVCaptureAudioDataOutput { + recorder?.appendAudio(sampleBuffer) + } + } +} diff --git a/ios/Classes/Camera/CameraPlugin.swift b/ios/Classes/Camera/CameraPlugin.swift new file mode 100644 index 0000000..44d7fd9 --- /dev/null +++ b/ios/Classes/Camera/CameraPlugin.swift @@ -0,0 +1,439 @@ +import AVFoundation +import Flutter +import UIKit + +/// `ux/camera` + `ux/camera/events` registrar. Routes channel calls +/// to per-handle [CameraInstance]s. Enforces device + audio claims +/// across instances so a second controller against an already-held +/// camera or audio session fails fast with the right error code. +/// +/// Phase 2 scope: no video recording. `startVideoRecording` / +/// `stopVideoRecording` return `unsupported_format`. +public class CameraPlugin: NSObject, NativePlugin, FlutterStreamHandler { + private weak var textureRegistry: FlutterTextureRegistry? + + private var instances: [Int: CameraInstance] = [:] + private var nextHandle: Int = 1 + + // Multi-instance contention bookkeeping. + private var devicesInUse: Set = [] + private var audioInUse: Bool = false + + private var eventSink: FlutterEventSink? + + public func register(with registrar: FlutterPluginRegistrar) { + textureRegistry = registrar.textures() + + let methods = FlutterMethodChannel( + name: "ux/camera", + binaryMessenger: registrar.messenger() + ) + methods.setMethodCallHandler { [weak self] call, result in + self?.handle(call, result: result) + } + + let events = FlutterEventChannel( + name: "ux/camera/events", + binaryMessenger: registrar.messenger() + ) + events.setStreamHandler(self) + } + + // MARK: - FlutterStreamHandler + + public func onListen( + withArguments arguments: Any?, + eventSink events: @escaping FlutterEventSink + ) -> FlutterError? { + eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + return nil + } + + // MARK: - Method dispatch + + private func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "availableCameras": + availableCameras(result: result) + + case "create": + create(args: call.arguments, result: result) + + case "initialize": + withInstance(call.arguments, result: result) { instance in + self.requestPermissions(audio: instance.audioClaimed) { granted, kind in + guard granted else { + result(FlutterError( + code: "permission_denied", + message: kind, + details: nil + )) + return + } + instance.sessionQueueAsync { + instance.initialize() + DispatchQueue.main.async { result(nil) } + } + } + } + + case "dispose": + withInstance(call.arguments, result: result) { instance in + // Release the device + audio claim *immediately* on .main + // so a follow-up `create` for the same camera can succeed + // even while the slow session teardown + // (stopRunning / removeInput / removeOutput) is still in + // flight on the per-instance sessionQueue. Two + // AVCaptureSessions can briefly coexist against the same + // device — AVFoundation serializes the underlying hardware + // access. Without this, a quick page-pop → re-push hits + // `device_busy` because the old claim hasn't been + // released yet. + self.releaseClaim(for: instance) + self.instances.removeValue(forKey: instance.handle) + instance.sessionQueueAsync { + instance.dispose() + DispatchQueue.main.async { result(nil) } + } + } + + case "setDescription": + guard let args = call.arguments as? [String: Any], + let cameraId = args["cameraId"] as? String else { + result(badArgs("setDescription")) + return + } + withInstance(args, result: result) { instance in + instance.sessionQueueAsync { + do { + let size = try instance.setDescription(cameraId: cameraId) + DispatchQueue.main.async { + result([ + "previewSize": [ + "width": size.width, + "height": size.height, + ] + ]) + } + } catch let error as NSError { + DispatchQueue.main.async { + result(FlutterError( + code: "init_failed", + message: error.localizedDescription, + details: nil + )) + } + } + } + } + + case "setFlashMode": + guard let args = call.arguments as? [String: Any], + let modeName = args["mode"] as? String else { + result(badArgs("setFlashMode")) + return + } + withInstance(args, result: result) { instance in + instance.sessionQueueAsync { + instance.setFlashMode(flashMode(from: modeName)) + DispatchQueue.main.async { result(nil) } + } + } + + case "lockCaptureOrientation": + guard let args = call.arguments as? [String: Any], + let raw = args["orientation"] as? String else { + result(badArgs("lockCaptureOrientation")) + return + } + withInstance(args, result: result) { instance in + let o = DeviceOrientationFlutter.parse(raw) + instance.sessionQueueAsync { + instance.lockCaptureOrientation(o) + DispatchQueue.main.async { result(nil) } + } + } + + case "unlockCaptureOrientation": + withInstance(call.arguments, result: result) { instance in + instance.sessionQueueAsync { + instance.unlockCaptureOrientation() + DispatchQueue.main.async { result(nil) } + } + } + + case "takePicture": + guard let args = call.arguments as? [String: Any], + let raw = args["snapshotOrientation"] as? String else { + result(badArgs("takePicture")) + return + } + withInstance(args, result: result) { instance in + let snapshot = DeviceOrientationFlutter.parse(raw) + instance.sessionQueueAsync { + instance.takePicture(snapshot: snapshot) { outcome in + switch outcome { + case .success(let path): + result(["path": path]) + case .failure(let error): + result(FlutterError( + code: "take_picture_failed", + message: error.localizedDescription, + details: nil + )) + } + } + } + } + + case "startVideoRecording": + guard let args = call.arguments as? [String: Any], + let raw = args["snapshotOrientation"] as? String else { + result(badArgs("startVideoRecording")) + return + } + withInstance(args, result: result) { instance in + let snapshot = DeviceOrientationFlutter.parse(raw) + instance.sessionQueueAsync { + do { + try instance.startVideoRecording(snapshot: snapshot) + DispatchQueue.main.async { result(nil) } + } catch let error as NSError { + DispatchQueue.main.async { + result(FlutterError( + code: "recorder_failed", + message: error.localizedDescription, + details: nil + )) + } + } + } + } + + case "stopVideoRecording": + withInstance(call.arguments, result: result) { instance in + instance.sessionQueueAsync { + instance.stopVideoRecording { outcome in + // `outcome` arrives on recorderQueue from + // VideoRecorder.deliver. Bounce to main for + // the FlutterResult. + DispatchQueue.main.async { + switch outcome { + case .success(let url): + result(["path": url.path]) + case .failure(let error): + result(FlutterError( + code: "recorder_failed", + message: error.localizedDescription, + details: nil + )) + } + } + } + } + } + + case "audioPermissionStatus": + let granted = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized + result(granted) + + case "openSettings": + DispatchQueue.main.async { + if let url = URL(string: UIApplication.openSettingsURLString) { + UIApplication.shared.open(url) + } + result(nil) + } + + default: + result(FlutterMethodNotImplemented) + } + } + + // MARK: - Method implementations + + private func availableCameras(result: @escaping FlutterResult) { + let cameras = CaptureDevice.discover().map { $0.toWire() } + result(cameras) + } + + private func create(args: Any?, result: @escaping FlutterResult) { + guard let args = args as? [String: Any], + let cameraId = args["cameraId"] as? String, + let enableAudio = args["enableAudio"] as? Bool else { + result(badArgs("create")) + return + } + guard let registry = textureRegistry else { + result(FlutterError( + code: "init_failed", + message: "Texture registry unavailable", + details: nil + )) + return + } + + if devicesInUse.contains(cameraId) { + result(FlutterError( + code: "device_busy", + message: cameraId, + details: nil + )) + return + } + if enableAudio && audioInUse { + result(FlutterError( + code: "audio_busy", + message: nil, + details: nil + )) + return + } + + let handle = nextHandle + nextHandle += 1 + + let instance = CameraInstance(handle: handle) + instance.onEvent = { [weak self] payload in + self?.eventSink?(payload) + } + instances[handle] = instance + devicesInUse.insert(cameraId) + if enableAudio { audioInUse = true } + + instance.sessionQueueAsync { + do { + try instance.create( + cameraId: cameraId, + enableAudio: enableAudio, + registry: registry + ) + let size = instance.previewSize + DispatchQueue.main.async { + result([ + "handle": handle, + "textureId": instance.textureId, + "previewSize": [ + "width": size.width, + "height": size.height, + ], + ]) + } + } catch let error as NSError { + DispatchQueue.main.async { + self.releaseClaim(for: instance) + self.instances.removeValue(forKey: handle) + result(FlutterError( + code: "init_failed", + message: error.localizedDescription, + details: nil + )) + } + } + } + } + + // MARK: - Helpers + + private func withInstance( + _ args: Any?, + result: @escaping FlutterResult, + body: @escaping (CameraInstance) -> Void + ) { + guard let map = args as? [String: Any], + let handle = (map["handle"] as? NSNumber)?.intValue else { + result(badArgs("missing handle")) + return + } + guard let instance = instances[handle] else { + result(FlutterError( + code: "disposed", + message: "No camera for handle \(handle)", + details: nil + )) + return + } + body(instance) + } + + private func releaseClaim(for instance: CameraInstance) { + if let id = instance.currentCameraId { + devicesInUse.remove(id) + } + if instance.audioClaimed { + audioInUse = false + } + } + + /// Request camera (always) and microphone (when [audio]) access. + /// Completion fires on `.main` with `(true, "")` on camera grant. + /// A denied microphone is **not** a fatal failure — the recording + /// pipeline tolerates a missing audio output cleanly (no audio + /// track in the file), and the user is more annoyed by "camera + /// won't even open" than by "video has no sound." Camera denial + /// is the only hard failure path here. + private func requestPermissions( + audio: Bool, + _ completion: @escaping (Bool, String) -> Void + ) { + requestAccess(for: .video) { videoGranted in + guard videoGranted else { + completion(false, "camera") + return + } + guard audio else { + completion(true, "") + return + } + // Prompt for mic if we haven't asked yet; result doesn't + // gate initialize either way. CameraInstance.configureSession + // already tolerates a missing mic (the audio device input + // throws on `AVCaptureDeviceInput(device:)`, the catch + // swallows, and audioDeviceInput stays nil — which + // [CameraInstance.startVideoRecording] reads as "skip + // adding an audio input to the writer"). + self.requestAccess(for: .audio) { _ in + completion(true, "") + } + } + } + + private func requestAccess( + for mediaType: AVMediaType, + _ completion: @escaping (Bool) -> Void + ) { + switch AVCaptureDevice.authorizationStatus(for: mediaType) { + case .authorized: + DispatchQueue.main.async { completion(true) } + case .notDetermined: + AVCaptureDevice.requestAccess(for: mediaType) { granted in + DispatchQueue.main.async { completion(granted) } + } + default: + DispatchQueue.main.async { completion(false) } + } + } +} + +// MARK: - Free functions + +private func badArgs(_ where_: String) -> FlutterError { + return FlutterError( + code: "bad_args", + message: "Bad arguments for \(where_)", + details: nil + ) +} + +private func flashMode(from raw: String) -> AVCaptureDevice.FlashMode { + switch raw { + case "always": return .on + case "off": return .off + default: return .off + } +} diff --git a/ios/Classes/Camera/CameraSession.swift b/ios/Classes/Camera/CameraSession.swift new file mode 100644 index 0000000..84ca352 --- /dev/null +++ b/ios/Classes/Camera/CameraSession.swift @@ -0,0 +1,122 @@ +import AVFoundation +import Foundation + +/// Thin wrapper around `AVCaptureSession` that owns the lifecycle +/// helpers and observes the runtime-error / interruption +/// notifications, surfacing them through closures so +/// [CameraInstance] doesn't repeat the boilerplate. +/// +/// All `AVCaptureSession` mutations (input/output add/remove, start, +/// stop) must run on the caller's `sessionQueue`. This class doesn't +/// enforce that; the contract is that +/// [CameraInstance.sessionQueue.async { … }] wraps every call site. +final class CameraSession { + let av: AVCaptureSession + + /// Called on `.main` when the session reports an unrecoverable + /// runtime error. Notable case: `.mediaServicesWereReset` — the + /// caller typically tears down and recreates the session. + var onRuntimeError: ((NSError) -> Void)? + + /// Called on `.main` when the session is interrupted (e.g. video + /// device taken by another foreground client, audio session + /// interruption, or app backgrounded with `usesApplicationAudioSession`). + /// String describes the reason — `"videoDeviceInUseByAnotherClient"`, + /// `"audioDeviceInUseByAnotherClient"`, `"videoDeviceNotAvailableInBackground"`, etc. + var onInterrupted: ((String) -> Void)? + + /// Called on `.main` when an earlier interruption ends. + var onResumed: (() -> Void)? + + private var runtimeErrorObserver: NSObjectProtocol? + private var interruptedObserver: NSObjectProtocol? + private var resumedObserver: NSObjectProtocol? + + init() { + av = AVCaptureSession() + + // Telegram + camera_avfoundation both set this — keeps + // AVFoundation from yanking our audio session category out + // from under the app. + av.automaticallyConfiguresApplicationAudioSession = false + av.usesApplicationAudioSession = true + + runtimeErrorObserver = NotificationCenter.default.addObserver( + forName: .AVCaptureSessionRuntimeError, + object: av, + queue: .main + ) { [weak self] note in + let error = note.userInfo?[AVCaptureSessionErrorKey] as? NSError + ?? NSError(domain: "ux.camera", code: -1) + self?.onRuntimeError?(error) + } + + interruptedObserver = NotificationCenter.default.addObserver( + forName: .AVCaptureSessionWasInterrupted, + object: av, + queue: .main + ) { [weak self] note in + let reason = note.userInfo?[AVCaptureSessionInterruptionReasonKey] + as? Int ?? 0 + self?.onInterrupted?(reasonName(for: reason)) + } + + resumedObserver = NotificationCenter.default.addObserver( + forName: .AVCaptureSessionInterruptionEnded, + object: av, + queue: .main + ) { [weak self] _ in + self?.onResumed?() + } + } + + deinit { + if let o = runtimeErrorObserver { NotificationCenter.default.removeObserver(o) } + if let o = interruptedObserver { NotificationCenter.default.removeObserver(o) } + if let o = resumedObserver { NotificationCenter.default.removeObserver(o) } + } + + /// Configure block; pairs `beginConfiguration` / + /// `commitConfiguration` so every add/remove batch lands as one + /// session update. Caller must be on sessionQueue. + func configure(_ block: () -> Void) { + av.beginConfiguration() + block() + av.commitConfiguration() + } + + /// Start the session if it isn't already running. + /// Caller must be on sessionQueue. + func start() { + if !av.isRunning { av.startRunning() } + } + + /// Stop the session if it's running. + /// Caller must be on sessionQueue. + func stop() { + if av.isRunning { av.stopRunning() } + } +} + +/// Decode the integer reason that comes with +/// `AVCaptureSessionWasInterrupted`. Used in the event payload sent +/// to Dart. +private func reasonName(for code: Int) -> String { + guard let reason = AVCaptureSession.InterruptionReason(rawValue: code) else { + return "unknown" + } + switch reason { + case .videoDeviceNotAvailableInBackground: + return "videoDeviceNotAvailableInBackground" + case .audioDeviceInUseByAnotherClient: + return "audioDeviceInUseByAnotherClient" + case .videoDeviceInUseByAnotherClient: + return "videoDeviceInUseByAnotherClient" + case .videoDeviceNotAvailableWithMultipleForegroundApps: + return "videoDeviceNotAvailableWithMultipleForegroundApps" + case .videoDeviceNotAvailableDueToSystemPressure: + return "videoDeviceNotAvailableDueToSystemPressure" + @unknown default: + return "unknown" + } +} diff --git a/ios/Classes/Camera/CaptureDevice.swift b/ios/Classes/Camera/CaptureDevice.swift new file mode 100644 index 0000000..e47f7c1 --- /dev/null +++ b/ios/Classes/Camera/CaptureDevice.swift @@ -0,0 +1,98 @@ +import AVFoundation + +/// Static helpers for camera-device discovery and per-device init. +/// +/// Mirrors what telegram-ios's `CameraDevice` does for the +/// session-priority + format-negotiation cases we actually care +/// about — front / back wide-angle only. Telegram's preference for +/// `TripleCamera` / `DualCamera` is for multi-cam zoom UX we don't +/// build today; if we ever need it, this is where it goes. +enum CaptureDevice { + /// Enumerate the front + back wide-angle cameras the system + /// exposes. Order: back first, front second. Stable for the + /// lifetime of the process — iOS doesn't hot-swap cameras. + static func discover() -> [DiscoveredCamera] { + let session = AVCaptureDevice.DiscoverySession( + deviceTypes: [.builtInWideAngleCamera], + mediaType: .video, + position: .unspecified + ) + // Sort so back devices come first; the chat composer opens + // the back camera by default elsewhere, so this matches the + // common "first available" pick. + return session.devices + .sorted { positionRank($0.position) < positionRank($1.position) } + .map { device in + DiscoveredCamera( + device: device, + lens: lensName(for: device.position), + // iOS doesn't expose sensor orientation directly; + // 90° matches what `camera_avfoundation` reports + // and what banlu's `normalizeCameraCapture` math + // assumes for iOS sensors. + sensorOrientation: 90 + ) + } + } + + /// Apply our default per-device config: continuous autofocus, + /// continuous auto-exposure, torch off. Idempotent; safe to call + /// repeatedly. The block is wrapped in + /// `lockForConfiguration` / `unlockForConfiguration`. + static func applyDefaults(_ device: AVCaptureDevice) { + do { + try device.lockForConfiguration() + defer { device.unlockForConfiguration() } + + if device.isFocusModeSupported(.continuousAutoFocus) { + device.focusMode = .continuousAutoFocus + } + if device.isExposureModeSupported(.continuousAutoExposure) { + device.exposureMode = .continuousAutoExposure + } + if device.hasTorch && device.isTorchModeSupported(.off) { + device.torchMode = .off + } + } catch { + // Best-effort — a device that refuses lockForConfiguration + // will still capture frames; we just can't tweak focus. + } + } + + // MARK: - private + + private static func positionRank(_ position: AVCaptureDevice.Position) -> Int { + switch position { + case .back: return 0 + case .front: return 1 + default: return 2 + } + } + + private static func lensName(for position: AVCaptureDevice.Position) -> String { + switch position { + case .front: return "front" + case .back: return "back" + default: return "back" + } + } +} + +/// One row of the discovery result. `lens` and `sensorOrientation` +/// match the wire shape expected by Dart's +/// `MethodChannelUxCameraBackend.availableCameras()`. +struct DiscoveredCamera { + let device: AVCaptureDevice + let lens: String + let sensorOrientation: Int + + var uniqueID: String { device.uniqueID } + + func toWire() -> [String: Any] { + return [ + "id": uniqueID, + "lens": lens, + "sensorOrientation": sensorOrientation, + ] + } +} diff --git a/ios/Classes/Camera/DeviceOrientationBridge.swift b/ios/Classes/Camera/DeviceOrientationBridge.swift new file mode 100644 index 0000000..6d21290 --- /dev/null +++ b/ios/Classes/Camera/DeviceOrientationBridge.swift @@ -0,0 +1,120 @@ +import AVFoundation +import UIKit + +/// Translates between Flutter's `DeviceOrientation` (4 enum values +/// shipped as strings across the channel) and AVFoundation's +/// `AVCaptureVideoOrientation`, and bridges physical-device rotation +/// notifications to a closure callback. +/// +/// Observed orientation source is `UIDevice.current.orientation` — +/// independent of any UI orientation lock, so this fires even while +/// the app's window is portrait-locked. `.faceUp` / `.faceDown` are +/// ignored (no useful direction). +/// +/// `UIDevice.beginGeneratingDeviceOrientationNotifications()` must be +/// called on main, balanced with `end…()`; this class enforces both. +final class DeviceOrientationBridge { + typealias Listener = (DeviceOrientationFlutter) -> Void + + private var listener: Listener? + private var observer: NSObjectProtocol? + + /// Most recent valid orientation observed. Initialised to + /// `portraitUp` so callers have a starting value before the first + /// rotation event. + private(set) var current: DeviceOrientationFlutter = .portraitUp + + /// Starts observing. Safe to call multiple times; subsequent calls + /// replace the listener but don't re-register. + func start(listener: @escaping Listener) { + self.listener = listener + guard observer == nil else { return } + + // beginGeneratingDeviceOrientationNotifications is main-only. + if Thread.isMainThread { + UIDevice.current.beginGeneratingDeviceOrientationNotifications() + } else { + DispatchQueue.main.sync { + UIDevice.current.beginGeneratingDeviceOrientationNotifications() + } + } + + // Seed `current` from the device's reported orientation if it + // is already a valid one. + if let seed = DeviceOrientationFlutter(uiDevice: UIDevice.current.orientation) { + current = seed + } + + observer = NotificationCenter.default.addObserver( + forName: UIDevice.orientationDidChangeNotification, + object: UIDevice.current, + queue: .main + ) { [weak self] _ in + guard let self = self else { return } + guard let next = DeviceOrientationFlutter(uiDevice: UIDevice.current.orientation) else { + return + } + guard next != self.current else { return } + self.current = next + self.listener?(next) + } + } + + func stop() { + listener = nil + if let observer = observer { + NotificationCenter.default.removeObserver(observer) + self.observer = nil + DispatchQueue.main.async { + UIDevice.current.endGeneratingDeviceOrientationNotifications() + } + } + } + + deinit { stop() } +} + +/// Mirrors Flutter's `DeviceOrientation` — the four cardinal values +/// that travel over the `ux/camera` channel as wire strings. `public` +/// so the host app's XCTest target can verify the +/// `DeviceOrientationFlutter` / `AVCaptureVideoOrientation` mapping +/// without `@testable import`. +public enum DeviceOrientationFlutter: String { + case portraitUp + case landscapeLeft + case portraitDown + case landscapeRight + + /// Parse a wire string. Returns `.portraitUp` for unknown inputs + /// (matches the Dart-side fallback in `MethodChannelUxCameraBackend`). + public static func parse(_ raw: String?) -> DeviceOrientationFlutter { + return DeviceOrientationFlutter(rawValue: raw ?? "") ?? .portraitUp + } + + /// `UIDeviceOrientation` → Flutter convention is a direct 1:1 by + /// name. Despite the AV-side flip, `UIDeviceOrientation.landscapeLeft` + /// and Flutter's `landscapeLeft` describe the same physical pose + /// (home button on the right). The flip lives in + /// [avVideoOrientation], not here. + public init?(uiDevice: UIDeviceOrientation) { + switch uiDevice { + case .portrait: self = .portraitUp + case .portraitUpsideDown: self = .portraitDown + case .landscapeLeft: self = .landscapeLeft + case .landscapeRight: self = .landscapeRight + default: return nil + } + } + + /// AVFoundation video orientation. Translates Flutter's portrait- + /// relative convention to AVFoundation's hardware-relative one. + /// Used to drive `AVCaptureConnection.videoOrientation`. + public var avVideoOrientation: AVCaptureVideoOrientation { + switch self { + case .portraitUp: return .portrait + case .portraitDown: return .portraitUpsideDown + case .landscapeLeft: return .landscapeRight + case .landscapeRight: return .landscapeLeft + } + } +} diff --git a/ios/Classes/Camera/PhotoOutput.swift b/ios/Classes/Camera/PhotoOutput.swift new file mode 100644 index 0000000..c93fb33 --- /dev/null +++ b/ios/Classes/Camera/PhotoOutput.swift @@ -0,0 +1,106 @@ +import AVFoundation +import Foundation + +/// Wraps `AVCapturePhotoOutput`. One instance per +/// [CameraInstance]; gets added to the session at create time and +/// stays for the session's lifetime (no swap on camera flip — the +/// output is generic, only the connection's video device changes). +/// +/// `take(orientation:flashMode:completion:)` is the only public entry. +/// It sets the photo connection's `videoOrientation` to the +/// snapshotted orientation just before firing, hands off to a +/// per-capture delegate, and resets the connection back to portrait +/// afterward (so a takePicture without an explicit snapshot — should +/// the path ever exist — falls back cleanly). +final class PhotoOutput { + let avOutput = AVCapturePhotoOutput() + + private var inFlight: PhotoCaptureDelegate? + + /// Capture a single still. [orientation] applies to the photo + /// connection. [flashMode] is applied to the per-shot + /// `AVCapturePhotoSettings`. [completion] is invoked on `.main` + /// with either the saved file path or an `NSError`. + func take( + orientation: DeviceOrientationFlutter, + flashMode: AVCaptureDevice.FlashMode, + completion: @escaping (Result) -> Void + ) { + guard let connection = avOutput.connection(with: .video) else { + completion(.failure(NSError( + domain: "ux.camera", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Photo connection unavailable"] + ))) + return + } + if connection.isVideoOrientationSupported { + connection.videoOrientation = orientation.avVideoOrientation + } + // The recorded photo carries no mirror; mirroring is a + // preview-only concern. + if connection.isVideoMirroringSupported { + connection.automaticallyAdjustsVideoMirroring = false + connection.isVideoMirrored = false + } + + let settings = AVCapturePhotoSettings() + if avOutput.supportedFlashModes.contains(flashMode) { + settings.flashMode = flashMode + } + + let delegate = PhotoCaptureDelegate { [weak self] result in + // Reset orientation on the photo connection so a future + // capture without a snapshot defaults to portrait. + if let conn = self?.avOutput.connection(with: .video), + conn.isVideoOrientationSupported { + conn.videoOrientation = .portrait + } + self?.inFlight = nil + DispatchQueue.main.async { completion(result) } + } + // Retain the delegate for the duration of the capture — + // AVCapturePhotoOutput holds it weakly. + inFlight = delegate + avOutput.capturePhoto(with: settings, delegate: delegate) + } +} + +/// Per-shot delegate. Receives the photo, writes +/// `fileDataRepresentation()` to a unique path under +/// `NSTemporaryDirectory()`, invokes the completion. The plugin +/// retains it via [PhotoOutput.inFlight] across the async hop. +private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate { + private let completion: (Result) -> Void + + init(completion: @escaping (Result) -> Void) { + self.completion = completion + } + + func photoOutput( + _ output: AVCapturePhotoOutput, + didFinishProcessingPhoto photo: AVCapturePhoto, + error: Error? + ) { + if let error = error as NSError? { + completion(.failure(error)) + return + } + guard let data = photo.fileDataRepresentation() else { + completion(.failure(NSError( + domain: "ux.camera", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "No photo data"] + ))) + return + } + let url = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("ux_camera_\(UUID().uuidString).jpg") + do { + try data.write(to: url, options: .atomic) + completion(.success(url.path)) + } catch let error as NSError { + completion(.failure(error)) + } + } +} diff --git a/ios/Classes/Camera/PreviewSink.swift b/ios/Classes/Camera/PreviewSink.swift new file mode 100644 index 0000000..2b30cd7 --- /dev/null +++ b/ios/Classes/Camera/PreviewSink.swift @@ -0,0 +1,66 @@ +import AVFoundation +import CoreVideo +import Flutter + +/// Single-slot latest-pixel-buffer sink that feeds a `FlutterTexture`. +/// +/// AVCaptureVideoDataOutput's sample-buffer delegate fires on +/// `videoBufferQueue`; we extract the `CVPixelBuffer`, store it as +/// `latestPixelBuffer`, and notify the texture registry on `main` so +/// Flutter pulls the new frame via `copyPixelBuffer()` on the engine +/// thread. +/// +/// We retain *only the most recent* buffer — the previous one is +/// released the moment a new sample arrives. This matches the +/// `camera_avfoundation` lifetime invariant and bounds memory at one +/// frame regardless of how fast we produce vs. consume. +final class PreviewSink: NSObject, FlutterTexture { + private weak var registry: FlutterTextureRegistry? + private var textureId: Int64 = -1 + + /// Serial queue for the `latestPixelBuffer` swap. Sample-buffer + /// delegate writes; `copyPixelBuffer()` reads. Without this the + /// pointer could be freed mid-read on a different thread. + private let bufferQueue = DispatchQueue( + label: "ux.camera.preview.buffer", + qos: .userInitiated + ) + private var latestPixelBuffer: CVPixelBuffer? + + func register(with registry: FlutterTextureRegistry) -> Int64 { + self.registry = registry + textureId = registry.register(self) + return textureId + } + + func unregister() { + registry?.unregisterTexture(textureId) + bufferQueue.sync { latestPixelBuffer = nil } + } + + // MARK: - FlutterTexture + + func copyPixelBuffer() -> Unmanaged? { + var pb: CVPixelBuffer? + bufferQueue.sync { + pb = latestPixelBuffer + latestPixelBuffer = nil + } + if let pb = pb { return Unmanaged.passRetained(pb) } + return nil + } + + /// Receives a new frame. Called from the + /// `AVCaptureVideoDataOutputSampleBufferDelegate` on + /// `videoBufferQueue`. Cheap — just swaps the pointer + notifies + /// the registry. + func receive(sampleBuffer: CMSampleBuffer) { + guard let pb = CMSampleBufferGetImageBuffer(sampleBuffer) else { + return + } + bufferQueue.sync { latestPixelBuffer = pb } + if let registry = registry { + registry.textureFrameAvailable(textureId) + } + } +} diff --git a/ios/Classes/Camera/VideoRecorder.swift b/ios/Classes/Camera/VideoRecorder.swift new file mode 100644 index 0000000..092cfbc --- /dev/null +++ b/ios/Classes/Camera/VideoRecorder.swift @@ -0,0 +1,563 @@ +import AVFoundation +import CoreMedia +import Foundation + +/// Owns one `AVAssetWriter`. Mirrors telegram-ios's +/// [VideoRecorder.swift](file:///Users/agra/projects/telegram-ios/submodules/Camera/Sources/VideoRecorder.swift) +/// state machine closely — lazy per-type input creation, +/// gated `startWriting`, pending-audio-buffer queue: +/// +/// 1. `start()` only creates the `AVAssetWriter` shell and sets +/// `recordingStartSampleTime` to wall-clock now. No inputs yet. +/// 2. First **video** sample → create `videoInput` from its +/// `CMFormatDescription` (`sourceFormatHint:`) + transform. +/// Pre-`startWriting` because audio input may still be pending. +/// 3. First **audio** sample (if `hasAudio`) → create `audioInput` +/// with sample-rate / channel-layout extracted from the audio +/// `CMFormatDescription` merged into `baseAudioSettings` +/// (`recommendedAudioSettingsForAssetWriter`). +/// 4. Next video sample arrives with both inputs added → +/// `assetWriter.startWriting()`. Sample is dropped (telegram's +/// behaviour — initial frame loss is acceptable, the writer needs +/// one cycle to settle). +/// 5. Subsequent video sample → `startSession(atSourceTime: pts)`, +/// `recordingStartSampleTime = pts`. Appends begin. +/// 6. Audio samples that arrive before `recordingStartSampleTime` is +/// set are queued in `pendingAudioSampleBuffers`. After each +/// successful video append, the queue is drained for samples whose +/// `endTime <= lastVideoSampleTime`. +/// 7. `stop()` sets `recordingStopSampleTime` to wall-clock now. +/// Sample callbacks set `hasAllVideoBuffers` / `hasAllAudioBuffers` +/// when their PTS crosses the stop time. `maybeFinish()` runs when +/// both flags are set, gates `finishWriting` on +/// `writer.status == .writing`. If audio never arrived, the audio +/// flag is set synchronously in `stop()` so the video side can +/// complete on its own. +/// +/// Sample-count diagnostics emit via [onDiagnostic] at each major +/// checkpoint so the operator can verify "audio actually captured" +/// without instrumenting the call sites. The closure is wired to the +/// Dart-side `Log.tag('camera').i(...)` via the `ux/camera/events` +/// channel — visible in `~/banlu/tools/log_server/data/banlu.jsonl`. +final class VideoRecorder { + /// Maps Flutter's `DeviceOrientation` to the rotation transform + /// embedded as `AVAssetWriterInput.transform`. Source buffers + /// are portrait-shape (see [CameraInstance.applyVideoOrientationOnPreview]), + /// so the table assumes portrait source — see + /// [CameraOrientationTests] for the four cases. + public static func transform( + for orientation: DeviceOrientationFlutter + ) -> CGAffineTransform { + switch orientation { + case .portraitUp: return .identity + case .portraitDown: return CGAffineTransform(rotationAngle: .pi) + case .landscapeLeft: return CGAffineTransform(rotationAngle: -.pi / 2) + case .landscapeRight: return CGAffineTransform(rotationAngle: .pi / 2) + } + } + + // MARK: - immutable config + + private let url: URL + private let videoTransform: CGAffineTransform + private let hasAudio: Bool + private let baseVideoSettings: [String: Any]? + private let baseAudioSettings: [String: Any] + private let recorderQueue: DispatchQueue + + // MARK: - mutable state (always touched on recorderQueue) + + private var writer: AVAssetWriter? + private var videoInput: AVAssetWriterInput? + private var audioInput: AVAssetWriterInput? + + /// Wall-clock "start" time set by [start], then overwritten to the + /// first video sample's PTS once the session is started. Used to + /// gate samples whose PTS is older than start. + private var recordingStartSampleTime: CMTime = .invalid + + /// Set by [stop]. Samples whose PTS crosses this set the matching + /// `hasAllXBuffers` flag and trigger [maybeFinish]. + private var recordingStopSampleTime: CMTime = .invalid + + /// PTS of the last video sample successfully appended. Used to + /// gate audio drains (audio samples whose `endTime` exceeds this + /// stay queued until video catches up). + private var lastVideoSampleTime: CMTime = .invalid + + private var startedSession = false + private var stopped = false + private var hasAllVideoBuffers = false + private var hasAllAudioBuffers = false + private var failed = false + + /// Audio samples arriving before video has caught up. Drained + /// after each successful video append. + private var pendingAudioSampleBuffers: [CMSampleBuffer] = [] + + private var completion: ((Result) -> Void)? + + // MARK: - diagnostics (emit via [onDiagnostic] → ux.Log) + + private func diag(_ message: String) { + onDiagnostic?(message) + } + + private var videoReceived: Int = 0 + private var videoAppended: Int = 0 + private var audioReceived: Int = 0 + private var audioAppended: Int = 0 + private var audioQueued: Int = 0 + + /// Set by [CameraInstance] to ship diagnostic messages over the + /// `ux/camera/events` channel as `{event: "diagnostic"}`. The + /// Dart-side controller turns those into `Log.tag('camera').i(...)` + /// — so they land in the log_server pipeline and can be tailed + /// from `~/banlu/tools/log_server/data/banlu.jsonl`. + var onDiagnostic: ((String) -> Void)? + + // MARK: - init / start + + init( + url: URL, + orientation: DeviceOrientationFlutter, + hasAudio: Bool, + baseVideoSettings: [String: Any]?, + baseAudioSettings: [String: Any], + recorderQueue: DispatchQueue + ) { + self.url = url + self.videoTransform = VideoRecorder.transform(for: orientation) + self.hasAudio = hasAudio + self.baseVideoSettings = baseVideoSettings + self.baseAudioSettings = baseAudioSettings + self.recorderQueue = recorderQueue + } + + /// Open the file. Inputs are created lazily on the first sample + /// of each type — see class doc. Throws on `AVAssetWriter` + /// allocation failure (typically a path / file-system issue). + func start() throws { + let writer = try AVAssetWriter(url: url, fileType: .mp4) + self.writer = writer + // Sentinel until the first video sample's PTS overwrites it — + // see [handleVideo] when it calls `writer.startSession`. + recordingStartSampleTime = CMTime( + seconds: CACurrentMediaTime(), + preferredTimescale: CMTimeScale(NSEC_PER_SEC) + ) + diag("start: file=\(url.lastPathComponent) hasAudio=\(hasAudio)") + } + + /// Hard cancel — drop pending audio, `cancelWriting` if the writer + /// is writing, delete the partial file. Mirrors telegram-ios's + /// [`VideoRecorder.cancelRecording`](file:///Users/agra/projects/telegram-ios/submodules/Camera/Sources/VideoRecorder.swift#L329). + /// Used by [CameraInstance.dispose] when a recording is in flight + /// at teardown — there's no caller to deliver the file to, so no + /// reason to wait for `finishWriting` to flush. + func cancel(completion: (() -> Void)? = nil) { + recorderQueue.async { + if self.stopped || self.failed { + completion?() + return + } + self.stopped = true + self.pendingAudioSampleBuffers = [] + if let writer = self.writer, writer.status == .writing { + writer.cancelWriting() + } + try? FileManager.default.removeItem(at: self.url) + self.diag("cancel: vRecv=\(self.videoReceived) aRecv=\(self.audioReceived)") + // Resolve any pending stop() completion so the caller's + // Future doesn't dangle. + if let cb = self.completion { + self.completion = nil + cb(.failure(NSError( + domain: "ux.camera", code: -12, + userInfo: [NSLocalizedDescriptionKey: "Recording cancelled"] + ))) + } + completion?() + } + } + + /// Stop. Sets `recordingStopSampleTime` so the next video / audio + /// sample crossing it flips the matching `hasAllXBuffers` flag, + /// which triggers `maybeFinish` → `finishWriting`. Completion + /// fires once when the writer finishes. + /// + /// Idempotent: a second call while a stop is already in flight is + /// silently dropped. + func stop(completion: @escaping (Result) -> Void) { + recorderQueue.async { + if self.completion != nil { return } + self.completion = completion + + let stopTime = CMTime( + seconds: CACurrentMediaTime(), + preferredTimescale: CMTimeScale(NSEC_PER_SEC) + ) + self.recordingStopSampleTime = stopTime + + self.diag("stop: vRecv=\(self.videoReceived) vApp=\(self.videoAppended)" + + " aRecv=\(self.audioReceived) aApp=\(self.audioAppended)" + + " aQueued=\(self.pendingAudioSampleBuffers.count)") + + // Nothing ever arrived — no sample callback will ever + // trigger `maybeFinish`. Cancel the writer instead. + if !self.startedSession { + self.writer?.cancelWriting() + self.failed = true + self.deliver(.failure(NSError( + domain: "ux.camera", code: -11, + userInfo: [ + NSLocalizedDescriptionKey: + "Recording stopped before any samples were written" + ] + ))) + return + } + + // No audio path (mic permission denied, etc.) — the audio + // side is "drained" by definition. `maybeFinish` then only + // waits for the next video sample whose PTS crosses + // `stopTime` (~one frame later, ~33ms at 30fps). + if self.audioInput == nil || self.audioReceived == 0 { + self.hasAllAudioBuffers = true + } + } + } + + // MARK: - sample append (from videoBufferQueue / audioBufferQueue) + + func appendVideo(_ sampleBuffer: CMSampleBuffer) { + recorderQueue.async { self.handleVideo(sampleBuffer) } + } + + func appendAudio(_ sampleBuffer: CMSampleBuffer) { + recorderQueue.async { self.handleAudio(sampleBuffer) } + } + + // MARK: - recorderQueue handlers + + private func handleVideo(_ sampleBuffer: CMSampleBuffer) { + guard !stopped, !failed else { return } + guard let writer = writer else { return } + guard + let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + CMFormatDescriptionGetMediaType(formatDescription) == kCMMediaType_Video + else { return } + + videoReceived += 1 + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + + // 1. Lazy create the video input on first video sample, with + // the buffer's format description as `sourceFormatHint`. + if videoInput == nil { + let videoSettings = baseVideoSettings ?? [:] + if writer.canApply(outputSettings: videoSettings, forMediaType: .video) { + let input = AVAssetWriterInput( + mediaType: .video, + outputSettings: videoSettings, + sourceFormatHint: formatDescription + ) + input.expectsMediaDataInRealTime = true + input.transform = videoTransform + if writer.canAdd(input) { + writer.add(input) + videoInput = input + diag("video input added") + } else { + fail(NSError(domain: "ux.camera", code: -30, + userInfo: [NSLocalizedDescriptionKey: "canAdd videoInput failed"])) + return + } + } else { + fail(NSError(domain: "ux.camera", code: -31, + userInfo: [NSLocalizedDescriptionKey: "canApply videoSettings failed"])) + return + } + } + + // 2. Writer state machine + if writer.status == .unknown { + // Drop samples that arrived BEFORE the wall-clock start + // (rare, but happens if the session was already running + // before start() was called). + if presentationTime < recordingStartSampleTime { + return + } + // Only start the writer when ALL needed inputs are ready. + if videoInput != nil && (audioInput != nil || !hasAudio) { + if !writer.startWriting() { + fail(writer.error) + return + } + diag("startWriting") + } + // Drop this sample regardless — the writer needs a cycle + // to settle. Next sample will hit the `.writing` branch. + return + } else if writer.status == .writing && !startedSession { + writer.startSession(atSourceTime: presentationTime) + recordingStartSampleTime = presentationTime + lastVideoSampleTime = presentationTime + startedSession = true + diag(String(format: "startSession at %.3fs", presentationTime.seconds)) + } + + // Drop pre-start samples (post-startSession). + if recordingStartSampleTime == .invalid + || presentationTime < recordingStartSampleTime { + return + } + + if writer.status == .writing && startedSession { + // 3. Stop-time gating — set hasAllVideoBuffers when we + // see a sample past stop time, trigger finish. + if recordingStopSampleTime.isValid + && presentationTime > recordingStopSampleTime { + hasAllVideoBuffers = true + maybeFinish() + return + } + + guard let input = videoInput else { return } + // Busy-wait briefly if the input isn't ready. Matches + // telegram-ios's pattern at VideoRecorder.swift:202-206. + // Real-time capture; we can't backpressure the camera. + while !input.isReadyForMoreMediaData { + RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.05)) + } + + if input.append(sampleBuffer) { + lastVideoSampleTime = presentationTime + videoAppended += 1 + } + + // 4. Drain any pending audio whose endTime now fits + // under lastVideoSampleTime. + if !tryAppendingPendingAudioBuffers() { + fail(writer.error) + } + } + } + + private func handleAudio(_ sampleBuffer: CMSampleBuffer) { + guard !stopped, !failed, hasAudio else { return } + guard let writer = writer else { return } + guard + let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + CMFormatDescriptionGetMediaType(formatDescription) == kCMMediaType_Audio + else { return } + + audioReceived += 1 + + // 1. Lazy create audio input on first audio sample, with + // sample-rate / channel-layout extracted from the + // sample's CMAudioFormatDescription. + if audioInput == nil { + var audioSettings = baseAudioSettings + + if let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) { + audioSettings[AVSampleRateKey] = asbd.pointee.mSampleRate + audioSettings[AVNumberOfChannelsKey] = asbd.pointee.mChannelsPerFrame + } + + var channelLayoutSize: Int = 0 + let channelLayoutPtr = CMAudioFormatDescriptionGetChannelLayout( + formatDescription, sizeOut: &channelLayoutSize + ) + let channelLayoutData: Data + if let ptr = channelLayoutPtr, channelLayoutSize > 0 { + channelLayoutData = Data(bytes: ptr, count: channelLayoutSize) + } else { + channelLayoutData = Data() + } + audioSettings[AVChannelLayoutKey] = channelLayoutData + + if writer.canApply(outputSettings: audioSettings, forMediaType: .audio) { + let input = AVAssetWriterInput( + mediaType: .audio, + outputSettings: audioSettings, + sourceFormatHint: formatDescription + ) + input.expectsMediaDataInRealTime = true + if writer.canAdd(input) { + writer.add(input) + audioInput = input + diag("audio input added" + + " sr=\(audioSettings[AVSampleRateKey] ?? "?")" + + " ch=\(audioSettings[AVNumberOfChannelsKey] ?? "?")") + } else { + diag("canAdd audioInput failed") + return + } + } else { + diag("canApply audioSettings failed") + return + } + } + + // 2. Need the video stream to have given us a session start + // time before any audio can be appended. + if recordingStartSampleTime == .invalid { return } + + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if presentationTime < recordingStartSampleTime { return } + + // 3. Stop-time gating. + if recordingStopSampleTime.isValid + && presentationTime > recordingStopSampleTime { + hasAllAudioBuffers = true + maybeFinish() + return + } + + // 4. Append (or queue) — drain pending first, then this + // sample. tryAppendingAudioSampleBuffer chooses queue vs + // immediate-append based on its endTime vs lastVideoSampleTime. + if !tryAppendingPendingAudioBuffers() + || !tryAppendingAudioSampleBuffer(sampleBuffer) { + fail(writer.error) + } + } + + // MARK: - audio buffer queue + + /// Append [sampleBuffer] immediately if its `endTime` doesn't + /// run past the latest video sample; otherwise enqueue. + private func tryAppendingAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer) -> Bool { + if sampleBuffer.endTime > lastVideoSampleTime { + pendingAudioSampleBuffers.append(sampleBuffer) + audioQueued += 1 + return true + } + return internalAppendAudioSampleBuffer(sampleBuffer) + } + + /// Drain queued audio samples that have caught up to the latest + /// video sample. Called after every video append. + private func tryAppendingPendingAudioBuffers() -> Bool { + guard !pendingAudioSampleBuffers.isEmpty else { return true } + + var stillPending: [CMSampleBuffer] = [] + stillPending.reserveCapacity(pendingAudioSampleBuffers.count) + var ok = true + for sample in pendingAudioSampleBuffers { + if !ok { + stillPending.append(sample) + continue + } + if sample.endTime <= lastVideoSampleTime { + if !internalAppendAudioSampleBuffer(sample) { + ok = false + } + } else { + stillPending.append(sample) + } + } + pendingAudioSampleBuffers = stillPending + return ok + } + + private func internalAppendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer) -> Bool { + guard startedSession, let input = audioInput else { return true } + while !input.isReadyForMoreMediaData { + RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.05)) + } + if input.append(sampleBuffer) { + audioAppended += 1 + return true + } + if writer?.error != nil { + return false + } + // Append returned false but no writer error — treat as + // recoverable. Telegram does the same. + return true + } + + // MARK: - finish + + private func maybeFinish() { + guard hasAllVideoBuffers, + (!hasAudio || hasAllAudioBuffers), + !stopped, !failed else { return } + stopped = true + finish() + } + + private func finish() { + // Drain any audio buffer still pending up to the stop time. + _ = tryAppendingPendingAudioBuffers() + + guard let writer = writer else { + deliver(.failure(NSError( + domain: "ux.camera", code: -40, + userInfo: [NSLocalizedDescriptionKey: "writer missing on finish"] + ))) + return + } + + // Only `finishWriting` when the writer reached `.writing`. + guard writer.status == .writing else { + diag("finish skipped — writer.status=\(writer.status.rawValue)") + failOnError(writer.error) + return + } + + let url = self.url + diag("finishWriting:" + + " vRecv=\(videoReceived) vApp=\(videoAppended)" + + " aRecv=\(audioReceived) aApp=\(audioAppended)" + + " aQueuedDrop=\(pendingAudioSampleBuffers.count)") + + writer.finishWriting { [weak self] in + self?.recorderQueue.async { + guard let self = self else { return } + if writer.status == .completed { + self.deliver(.success(url)) + } else { + self.failOnError(writer.error) + } + } + } + } + + private func fail(_ error: Error?) { + if failed { return } + failed = true + failOnError(error) + } + + private func failOnError(_ error: Error?) { + try? FileManager.default.removeItem(at: url) + let ns = (error as NSError?) ?? NSError( + domain: "ux.camera", code: -41, + userInfo: [NSLocalizedDescriptionKey: "AVAssetWriter failed"] + ) + deliver(.failure(ns)) + } + + private func deliver(_ outcome: Result) { + let cb = completion + completion = nil + cb?(outcome) + } +} + +// MARK: - CMSampleBuffer ergonomics + +private extension CMSampleBuffer { + /// `presentationTime + duration` — last instant covered by this + /// buffer. Used to pace audio against video. + var endTime: CMTime { + let pts = CMSampleBufferGetPresentationTimeStamp(self) + let dur = CMSampleBufferGetDuration(self) + if dur.flags.contains(.valid) { + return pts + dur + } + return pts + } +} diff --git a/ios/Classes/UxPlugin.swift b/ios/Classes/UxPlugin.swift index 4da56a3..e9625c7 100644 --- a/ios/Classes/UxPlugin.swift +++ b/ios/Classes/UxPlugin.swift @@ -13,6 +13,7 @@ public class UxPlugin: NSObject, FlutterPlugin { ClipboardPlugin(), GalleryPlugin(), CrashPlugin(), + CameraPlugin(), ] for plugin in plugins { plugin.register(with: registrar) diff --git a/ios/ux.podspec b/ios/ux.podspec index 5d072be..133a247 100644 --- a/ios/ux.podspec +++ b/ios/ux.podspec @@ -1,13 +1,13 @@ Pod::Spec.new do |s| s.name = 'ux' s.version = '0.9.0' - s.summary = 'UX Kit — Flutter plugin: keyboard, sensor, file, and QR scanner.' + s.summary = 'UX Kit — Flutter plugin: keyboard, sensor, file, QR scanner, and camera.' s.homepage = 'https://swipelab.co/ux.html' s.license = { :file => '../LICENSE' } s.author = { 'Swipelab' => 'hello@swipelab.co' } s.source = { :path => '.' } s.source_files = 'Classes/**/*.swift' - s.frameworks = ['Photos', 'PhotosUI'] + s.frameworks = ['AVFoundation', 'CoreMedia', 'CoreVideo', 'Photos', 'PhotosUI'] s.dependency 'Flutter' s.ios.deployment_target = '13.0' s.swift_version = '5.0' diff --git a/lib/src/camera/camera.dart b/lib/src/camera/camera.dart index 8abebff..99e4ebc 100644 --- a/lib/src/camera/camera.dart +++ b/lib/src/camera/camera.dart @@ -6,10 +6,13 @@ import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart' show Widget; import '../file.dart' show UxFile; +import '../log.dart' show Log; import '../sensor.dart' show UxSensor; import 'camera_backend.dart'; import 'camera_preview.dart' show UxCameraPreview; +final _log = Log.tag('camera'); + /// Describes a camera device on the system. Returned by /// [uxAvailableCameras]; passed to [UxCameraController] to bind a /// specific lens. @@ -67,6 +70,7 @@ class UxCameraValue { this.isRecordingVideo = false, this.deviceOrientation = DeviceOrientation.portraitUp, this.enableAudio = false, + this.audioPermissionGranted = false, this.errorDescription, }); @@ -91,6 +95,14 @@ class UxCameraValue { final bool enableAudio; + /// True iff the user has granted microphone access. Updated when + /// the controller initialises and on + /// [UxCameraController.refreshAudioPermission]. Independent of + /// [enableAudio] — a controller can request audio (`enableAudio: + /// true`) without having permission, in which case recordings have + /// no audio track and callers should surface a hint. + final bool audioPermissionGranted; + /// Set to the last native session error's message when one fires. /// Cleared on the next successful state transition. final String? errorDescription; @@ -104,6 +116,7 @@ class UxCameraValue { bool? isRecordingVideo, DeviceOrientation? deviceOrientation, bool? enableAudio, + bool? audioPermissionGranted, Object? errorDescription = _unset, }) => UxCameraValue( @@ -113,6 +126,7 @@ class UxCameraValue { isRecordingVideo: isRecordingVideo ?? this.isRecordingVideo, deviceOrientation: deviceOrientation ?? this.deviceOrientation, enableAudio: enableAudio ?? this.enableAudio, + audioPermissionGranted: audioPermissionGranted ?? this.audioPermissionGranted, errorDescription: identical(errorDescription, _unset) ? this.errorDescription : errorDescription as String?, @@ -177,24 +191,37 @@ class UxCameraController extends ValueNotifier { /// instance already holds the requested device or the audio session. Future initialize() async { _throwIfDisposed('initialize'); - final result = await UxCameraBackend.instance.create( - cameraId: description.id, - enableAudio: value.enableAudio, - preset: resolutionPreset, - ); - _handle = result.handle; - _textureId = result.textureId; - _eventsSub = UxCameraBackend.instance.events(result.handle).listen( - _onEvent, - onError: (Object error, StackTrace? stack) { - value = value.copyWith(errorDescription: error.toString()); - }, - ); - await UxCameraBackend.instance.initialize(result.handle); - value = value.copyWith( - isInitialized: true, - previewSize: result.previewSize, - ); + try { + final result = await UxCameraBackend.instance.create( + cameraId: description.id, + enableAudio: value.enableAudio, + preset: resolutionPreset, + ); + _handle = result.handle; + _textureId = result.textureId; + _eventsSub = UxCameraBackend.instance.events(result.handle).listen( + _onEvent, + onError: (Object error, StackTrace? stack) { + value = value.copyWith(errorDescription: error.toString()); + }, + ); + await UxCameraBackend.instance.initialize(result.handle); + final audioGranted = + await UxCameraBackend.instance.audioPermissionGranted(); + value = value.copyWith( + isInitialized: true, + previewSize: result.previewSize, + audioPermissionGranted: audioGranted, + ); + } catch (_) { + // initialize failed mid-way. The native side may have allocated + // a handle + claimed the camera/audio. Tear down what we have + // so the next attempt isn't blocked by a leaked device claim. + // [_handle] / [_eventsSub] are cleaned by [dispose] which + // tolerates the partial state. + await dispose(); + rethrow; + } } void _onEvent(UxCameraEvent event) { @@ -205,8 +232,9 @@ class UxCameraController extends ValueNotifier { value = value.copyWith(errorDescription: description ?? code); case UxCameraSessionInterrupted(): case UxCameraSessionResumed(): - // Lifecycle pings; recovery is automatic on the native side. break; + case UxCameraDiagnostic(:final message): + _log.i('recorder: $message'); } } @@ -278,6 +306,31 @@ class UxCameraController extends ValueNotifier { return file; } + /// Re-poll the OS for mic permission state and update + /// [value.audioPermissionGranted]. Call on + /// `AppLifecycleState.resumed` to pick up grants made via Settings. + Future refreshAudioPermission() async { + _throwIfDisposed('refreshAudioPermission'); + final granted = await UxCameraBackend.instance.audioPermissionGranted(); + if (granted != value.audioPermissionGranted) { + value = value.copyWith(audioPermissionGranted: granted); + } + } + + /// Whether the user has granted mic permission to the app. Static + /// because the answer is global to the process — useful from UI that + /// needs the status before any controller has been created (e.g. + /// the camera page's "Tap to enable mic" banner). + static Future audioPermissionGranted() => + UxCameraBackend.instance.audioPermissionGranted(); + + /// Deep-link into the system Settings page so the user can grant + /// mic permission. Static because it doesn't depend on any active + /// controller — useful from the banner tap before the controller + /// has finished initialising. + static Future openSystemSettings() => + UxCameraBackend.instance.openSettings(); + /// Texture-backed widget that renders the live preview at its parent's /// size. Hero-flightable. Widget buildPreview() => UxCameraPreview(controller: this); diff --git a/lib/src/camera/camera_backend.dart b/lib/src/camera/camera_backend.dart index e425d05..b87d89d 100644 --- a/lib/src/camera/camera_backend.dart +++ b/lib/src/camera/camera_backend.dart @@ -85,6 +85,15 @@ abstract class UxCameraBackend { /// controller subscribes during [initialize] and unsubscribes on /// [disposeInstance]. Stream events(int handle); + + /// True iff the user has granted microphone access. Cheap; safe to + /// re-poll on app foregrounding to detect grants made via Settings. + Future audioPermissionGranted(); + + /// Deep-link into the system Settings page for this app. Caller is + /// expected to refresh [audioPermissionGranted] on + /// `AppLifecycleState.resumed`. + Future openSettings(); } /// The tuple returned by [UxCameraBackend.create] — everything the @@ -128,3 +137,11 @@ class UxCameraSessionInterrupted extends UxCameraEvent { class UxCameraSessionResumed extends UxCameraEvent { const UxCameraSessionResumed(super.handle); } + +/// Free-text diagnostic message from the native recorder. Routed by +/// the controller to `Log.tag('camera').i(...)` so it lands in the +/// log_server pipeline (`~/banlu/tools/log_server/data/banlu.jsonl`). +class UxCameraDiagnostic extends UxCameraEvent { + const UxCameraDiagnostic(super.handle, this.message); + final String message; +} diff --git a/lib/src/camera/camera_channel.dart b/lib/src/camera/camera_channel.dart index 239169d..3d1e888 100644 --- a/lib/src/camera/camera_channel.dart +++ b/lib/src/camera/camera_channel.dart @@ -131,6 +131,12 @@ class MethodChannelUxCameraBackend implements UxCameraBackend { return UxFile(m['path'] as String); } + @override + Future audioPermissionGranted() => _invoke('audioPermissionStatus'); + + @override + Future openSettings() => _invokeVoid('openSettings'); + @override Stream events(int handle) { return _rawEvents @@ -162,6 +168,11 @@ class MethodChannelUxCameraBackend implements UxCameraBackend { ); case 'sessionResumed': return UxCameraSessionResumed(handle); + case 'diagnostic': + return UxCameraDiagnostic( + handle, + m['message'] as String? ?? '', + ); default: return UxCameraSessionError(handle, 'unknown_event', null); } diff --git a/lib/src/testing/fake_camera.dart b/lib/src/testing/fake_camera.dart index eca9f77..e082dda 100644 --- a/lib/src/testing/fake_camera.dart +++ b/lib/src/testing/fake_camera.dart @@ -60,6 +60,12 @@ class FakeUxCameraBackend implements UxCameraBackend { UxCameraException? startVideoRecordingError; UxCameraException? stopVideoRecordingError; + /// Audio permission state returned by [audioPermissionGranted]. + /// Tests mutate this to drive the mic-permission UI banner. + bool audioPermission = true; + int audioPermissionCalls = 0; + int openSettingsCalls = 0; + // ---- internal --------------------------------------------------- int _nextHandle = 1; @@ -94,6 +100,10 @@ class FakeUxCameraBackend implements UxCameraBackend { _controllerFor(handle).add(UxCameraSessionResumed(handle)); } + void emitDiagnostic(int handle, String message) { + _controllerFor(handle).add(UxCameraDiagnostic(handle, message)); + } + // ---- UxCameraBackend impl -------------------------------------- @override @@ -195,4 +205,15 @@ class FakeUxCameraBackend implements UxCameraBackend { @override Stream events(int handle) => _controllerFor(handle).stream; + + @override + Future audioPermissionGranted() async { + audioPermissionCalls += 1; + return audioPermission; + } + + @override + Future openSettings() async { + openSettingsCalls += 1; + } } diff --git a/lib/ux.dart b/lib/ux.dart index 191991c..b9c5e3a 100644 --- a/lib/ux.dart +++ b/lib/ux.dart @@ -9,7 +9,7 @@ export 'src/bend_box.dart'; export 'src/json_extension.dart'; export 'src/bezier.dart'; export 'src/camera/camera.dart'; -export 'src/camera/camera_backend.dart' show UxCameraBackend, UxCameraCreateResult, UxCameraEvent, UxCameraDeviceOrientationChanged, UxCameraSessionError, UxCameraSessionInterrupted, UxCameraSessionResumed; +export 'src/camera/camera_backend.dart' show UxCameraBackend, UxCameraCreateResult, UxCameraEvent, UxCameraDeviceOrientationChanged, UxCameraSessionError, UxCameraSessionInterrupted, UxCameraSessionResumed, UxCameraDiagnostic; export 'src/camera/camera_channel.dart' show MethodChannelUxCameraBackend; export 'src/camera/camera_preview.dart'; export 'src/clipboard.dart'; diff --git a/test/camera/camera_channel_test.dart b/test/camera/camera_channel_test.dart index 6a83b9d..5347ea3 100644 --- a/test/camera/camera_channel_test.dart +++ b/test/camera/camera_channel_test.dart @@ -152,6 +152,60 @@ void main() { expect(file.path, '/tmp/v.mp4'); }); + test('events stream filters by handle and decodes diagnostic events', + () async { + const eventsChannel = EventChannel('ux/camera/events'); + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + + messenger.setMockStreamHandler( + eventsChannel, + MockStreamHandler.inline( + onListen: (_, sink) { + sink.success({ + 'event': 'diagnostic', + 'handle': 4, + 'message': 'video input added', + }); + sink.success({ + 'event': 'diagnostic', + 'handle': 7, + 'message': 'audio input added sr=44100 ch=1', + }); + sink.endOfStream(); + }, + ), + ); + addTearDown(() => messenger.setMockStreamHandler(eventsChannel, null)); + + final received = []; + await backend.events(4).forEach(received.add); + + expect(received, hasLength(1)); + final e = received.single as UxCameraDiagnostic; + expect(e.handle, 4); + expect(e.message, 'video input added'); + }); + + test('audioPermissionStatus + openSettings round-trip', () async { + var permissionReply = true; + handle((call) { + if (call.method == 'audioPermissionStatus') return permissionReply; + if (call.method == 'openSettings') return null; + return null; + }); + + expect(await backend.audioPermissionGranted(), isTrue); + permissionReply = false; + expect(await backend.audioPermissionGranted(), isFalse); + await backend.openSettings(); + + expect( + calls.map((c) => c.method).toList(), + ['audioPermissionStatus', 'audioPermissionStatus', 'openSettings'], + ); + }); + test('PlatformException maps to UxCameraException carrying code/message', () async { handle((_) => throw PlatformException( diff --git a/test/camera/camera_controller_test.dart b/test/camera/camera_controller_test.dart index f8650e2..242c822 100644 --- a/test/camera/camera_controller_test.dart +++ b/test/camera/camera_controller_test.dart @@ -68,6 +68,31 @@ void main() { expect(ctrl.value.deviceOrientation, DeviceOrientation.portraitDown); }); + test('diagnostic events route through Log.tag("camera") and do not ' + 'mutate value', () async { + final records = []; + final prevSink = Log.sink; + Log.configure( + minLevel: LogLevel.info, + sink: _CapturingSink(records), + captureCrashes: () {}, + ); + addTearDown(() => Log.configure(sink: prevSink, captureCrashes: () {})); + + final ctrl = UxCameraController(_front, UxResolutionPreset.high, enableAudio: false); + addTearDown(ctrl.dispose); + await ctrl.initialize(); + + final beforeValue = ctrl.value; + fake.emitDiagnostic(1, 'video input added'); + await Future.delayed(Duration.zero); + + expect(ctrl.value, beforeValue); + final diag = records.singleWhere((r) => r.tag == 'camera'); + expect(diag.level, LogLevel.info); + expect(diag.message, 'recorder: video input added'); + }); + test('sessionError events surface as value.errorDescription', () async { final ctrl = UxCameraController(_front, UxResolutionPreset.high, enableAudio: false); addTearDown(ctrl.dispose); @@ -216,6 +241,36 @@ void main() { expect(b.value.deviceOrientation, DeviceOrientation.landscapeRight); }); + test('initialize captures audioPermissionGranted into value', () async { + fake.audioPermission = false; + final ctrl = UxCameraController(_front, UxResolutionPreset.high, enableAudio: true); + addTearDown(ctrl.dispose); + + await ctrl.initialize(); + + expect(ctrl.value.audioPermissionGranted, isFalse); + expect(fake.audioPermissionCalls, 1); + }); + + test('refreshAudioPermission re-polls and updates value', () async { + fake.audioPermission = false; + final ctrl = UxCameraController(_front, UxResolutionPreset.high, enableAudio: true); + addTearDown(ctrl.dispose); + await ctrl.initialize(); + expect(ctrl.value.audioPermissionGranted, isFalse); + + fake.audioPermission = true; + await ctrl.refreshAudioPermission(); + + expect(ctrl.value.audioPermissionGranted, isTrue); + expect(fake.audioPermissionCalls, 2); + }); + + test('openSystemSettings dispatches to the backend', () async { + await UxCameraController.openSystemSettings(); + expect(fake.openSettingsCalls, 1); + }); + test('initialize propagates UxCameraException("permission_denied")', () async { fake.createError = const UxCameraException('permission_denied', 'camera'); final ctrl = UxCameraController(_front, UxResolutionPreset.high, enableAudio: false); @@ -225,3 +280,15 @@ void main() { throwsA(isA().having((e) => e.code, 'code', 'permission_denied'))); }); } + +class _CapturingSink extends LogSink { + _CapturingSink(this.records); + + final List records; + + @override + LogLevel get minLevel => LogLevel.trace; + + @override + void emit(LogRecord record) => records.add(record); +}