cb2b57f661
Per Apple's macOS AVCaptureSession.h docs (line 1106), setting `videoRotationAngle` on `AVCapturePhotoOutput`'s connection "does not necessarily result in physical rotation of video buffers … In the AVCapturePhotoOutput, orientation is handled using Exif tags." So our connection-rotation tweaks only affect the EXIF Orientation tag the JPEG carries — pixel data is sensor-native. Yet the user keeps seeing a rotated JPEG even after `stripJpegApp1` removes APP1 (EXIF). So either the pixel buffer IS rotated despite the docs, or EXIF is in a non-APP1 marker, or Flutter's decoder auto-rotates somehow. Log the actual captured JPEG's dimensions + EXIF Orientation to banlu.jsonl via the existing per-handle diagnostic stream: `CGImageSourceCopyPropertiesAtIndex` reads `kCGImagePropertyPixelWidth/Height/Orientation` from the JPEG bytes that `AVCapturePhoto.fileDataRepresentation()` produces. Format: `photo: WxH landscape|portrait exifOrientation=N`. Once we see what AVCapturePhotoOutput is actually producing on the user's Mac we'll know which side of the pipeline to fix.
560 lines
20 KiB
Swift
560 lines
20 KiB
Swift
import AVFoundation
|
||
#if canImport(UIKit)
|
||
import Flutter
|
||
#else
|
||
import FlutterMacOS
|
||
#endif
|
||
import Foundation
|
||
|
||
/// One per `UxCameraController` on the Dart side. Owns its
|
||
/// `AVCaptureSession`, the texture-backed preview pipeline, photo
|
||
/// output, audio + video data outputs, and the
|
||
/// [VideoRecorder] when a recording is in flight. Multiple instances
|
||
/// coexist; the `[CameraPlugin]` keys them by `handle`.
|
||
///
|
||
/// **Threading**:
|
||
/// - Session config (add/remove inputs/outputs, start, stop) runs on
|
||
/// `sessionQueue` (serial).
|
||
/// - Video sample-buffer delegate fires on `videoBufferQueue` (serial).
|
||
/// - Audio sample-buffer delegate fires on `audioBufferQueue` (serial).
|
||
/// - [VideoRecorder] mutates its writer state on `recorderQueue` (serial).
|
||
/// - All public completions land on `.main`.
|
||
final class CameraInstance {
|
||
let handle: Int
|
||
|
||
/// Called whenever the instance has an event to forward to Dart.
|
||
/// The payload is the `{event, handle, …}` map the EventChannel
|
||
/// emits. Set by the plugin at construction.
|
||
var onEvent: (([String: Any]) -> Void)?
|
||
|
||
private let session = CameraSession()
|
||
private let sink = PreviewSink()
|
||
private let photoOutput = PhotoOutput()
|
||
private let orientation = DeviceOrientationBridge()
|
||
|
||
private let sessionQueue: DispatchQueue
|
||
private let videoBufferQueue: DispatchQueue
|
||
private let audioBufferQueue: DispatchQueue
|
||
private let recorderQueue: DispatchQueue
|
||
|
||
private var device: AVCaptureDevice?
|
||
private var deviceInput: AVCaptureDeviceInput?
|
||
private var videoDataOutput: AVCaptureVideoDataOutput?
|
||
|
||
private var audioDevice: AVCaptureDevice?
|
||
private var audioDeviceInput: AVCaptureDeviceInput?
|
||
private var audioDataOutput: AVCaptureAudioDataOutput?
|
||
|
||
private let fanout: SampleFanout
|
||
|
||
private var flashMode: AVCaptureDevice.FlashMode = .off
|
||
private var lockedOrientation: DeviceOrientationFlutter?
|
||
private var enableAudio: Bool = false
|
||
private var disposed = false
|
||
|
||
/// Set during [startVideoRecording], cleared after the stop
|
||
/// completion fires. Lives on `sessionQueue` (set/cleared); the
|
||
/// [SampleFanout] holds a parallel reference (under its own lock)
|
||
/// for the videoBufferQueue / audioBufferQueue to access without
|
||
/// crossing queue boundaries per frame.
|
||
private var videoRecorder: VideoRecorder?
|
||
|
||
/// `uniqueID` of the AVCaptureDevice this instance is currently
|
||
/// bound to, or `nil` after [dispose]. Used by [CameraPlugin] to
|
||
/// release its device claim. Read on `.main`.
|
||
var currentCameraId: String? { device?.uniqueID }
|
||
|
||
/// Whether this instance was constructed with `enableAudio: true`
|
||
/// and therefore owns the app-global audio claim. Read on `.main`.
|
||
var audioClaimed: Bool { enableAudio }
|
||
|
||
/// Hop to this instance's serial session queue. Public entry
|
||
/// point so the plugin can dispatch session work without
|
||
/// exposing the queue directly.
|
||
func sessionQueueAsync(_ block: @escaping () -> Void) {
|
||
sessionQueue.async(execute: block)
|
||
}
|
||
|
||
/// Active-format dimensions in the camera sensor's natural
|
||
/// orientation (typically landscape — `1920×1080` etc.). Set
|
||
/// after `create` / `setDescription` configures the session.
|
||
private(set) var previewSize: CGSize = .zero
|
||
|
||
/// Texture id handed back to Dart. Stable for the lifetime of
|
||
/// the instance.
|
||
private(set) var textureId: Int64 = -1
|
||
|
||
init(handle: Int) {
|
||
self.handle = handle
|
||
sessionQueue = DispatchQueue(label: "ux.camera.session.\(handle)")
|
||
videoBufferQueue = DispatchQueue(label: "ux.camera.video.\(handle)")
|
||
audioBufferQueue = DispatchQueue(label: "ux.camera.audio.\(handle)")
|
||
recorderQueue = DispatchQueue(label: "ux.camera.recorder.\(handle)")
|
||
fanout = SampleFanout(sink: sink)
|
||
|
||
session.onRuntimeError = { [weak self] error in
|
||
self?.emit([
|
||
"event": "sessionError",
|
||
"code": "session_runtime_error",
|
||
"description": error.localizedDescription,
|
||
])
|
||
}
|
||
session.onInterrupted = { [weak self] reason in
|
||
self?.emit(["event": "sessionInterrupted", "reason": reason])
|
||
}
|
||
|
||
photoOutput.onCapturedDiagnostic = { [weak self] message in
|
||
self?.emit([
|
||
"event": "diagnostic",
|
||
"message": message,
|
||
])
|
||
}
|
||
session.onResumed = { [weak self] in
|
||
self?.emit(["event": "sessionResumed"])
|
||
}
|
||
}
|
||
|
||
// MARK: - Lifecycle
|
||
|
||
/// Synchronously configure the session for [cameraId]. Registers
|
||
/// the texture, attaches audio if requested, upgrades the audio
|
||
/// session, and starts the orientation bridge. Must run on
|
||
/// sessionQueue.
|
||
func create(
|
||
cameraId: String,
|
||
enableAudio: Bool,
|
||
registry: FlutterTextureRegistry
|
||
) throws {
|
||
precondition(!disposed)
|
||
self.enableAudio = enableAudio
|
||
|
||
textureId = sink.register(with: registry)
|
||
|
||
if enableAudio {
|
||
// Widen the shared audio session category before we
|
||
// attach the mic input — matches `camera_avfoundation`'s
|
||
// defensive pattern. No-op if already widened.
|
||
AudioSession.upgradeForRecording()
|
||
}
|
||
|
||
try configureSession(forDeviceUniqueID: cameraId, replacing: false)
|
||
|
||
// The active-format dims from `device.activeFormat` don't
|
||
// always match the buffer the data output actually delivers
|
||
// (notably on macOS, where the session preset gets remapped
|
||
// mid-pipeline). Forward the real first-frame size to Dart
|
||
// so camera_thumb's SizedBox aspect matches the texture's
|
||
// actual aspect — without this the FittedBox(cover) stretches
|
||
// a 16:9 buffer that's been sized as 4:3.
|
||
sink.onFirstFrameSize = { [weak self] size in
|
||
self?.emit([
|
||
"event": "previewSizeChanged",
|
||
"previewSize": [
|
||
"width": size.width,
|
||
"height": size.height,
|
||
],
|
||
])
|
||
}
|
||
|
||
orientation.start { [weak self] next in
|
||
guard let self = self else { return }
|
||
self.sessionQueue.async { self.applyOrientationFollowDevice(next) }
|
||
self.emit([
|
||
"event": "deviceOrientationChanged",
|
||
"orientation": next.rawValue,
|
||
])
|
||
}
|
||
}
|
||
|
||
/// Start the session. Must run on sessionQueue.
|
||
func initialize() {
|
||
precondition(!disposed)
|
||
session.start()
|
||
}
|
||
|
||
/// Tear everything down. Idempotent. Hard-cancels any in-flight
|
||
/// recording (drops queued audio, `cancelWriting`, deletes the
|
||
/// partial file — telegram-ios's `cancelRecording` path). Must run
|
||
/// on sessionQueue.
|
||
///
|
||
/// Order matters for releasing the camera back to the OS:
|
||
/// 1. Cancel the recorder so its retain on the audio data output
|
||
/// drops.
|
||
/// 2. Stop the session so the hardware stops streaming.
|
||
/// 3. Detach sample-buffer delegates — `setSampleBufferDelegate`
|
||
/// holds a strong reference to the delegate (our `SampleFanout`),
|
||
/// which transitively keeps the session alive on macOS.
|
||
/// 4. Remove inputs + outputs inside a single
|
||
/// `beginConfiguration` / `commitConfiguration` so AVFoundation
|
||
/// sees one atomic teardown rather than a sequence of partial
|
||
/// states. macOS's `AVCaptureDevice.DiscoverySession` won't
|
||
/// return a device that's still claimed by a half-torn-down
|
||
/// session, which is what was causing "camera not found" on
|
||
/// re-open after a few use cycles.
|
||
func dispose() {
|
||
if disposed { return }
|
||
disposed = true
|
||
|
||
if let recorder = videoRecorder {
|
||
recorder.cancel()
|
||
videoRecorder = nil
|
||
fanout.recorder = nil
|
||
}
|
||
|
||
session.stop()
|
||
|
||
videoDataOutput?.setSampleBufferDelegate(nil, queue: nil)
|
||
audioDataOutput?.setSampleBufferDelegate(nil, queue: nil)
|
||
|
||
session.configure {
|
||
if let input = deviceInput { session.av.removeInput(input) }
|
||
if let input = audioDeviceInput { session.av.removeInput(input) }
|
||
if let output = videoDataOutput { session.av.removeOutput(output) }
|
||
if let output = audioDataOutput { session.av.removeOutput(output) }
|
||
session.av.removeOutput(photoOutput.avOutput)
|
||
}
|
||
|
||
videoDataOutput = nil
|
||
deviceInput = nil
|
||
device = nil
|
||
audioDeviceInput = nil
|
||
audioDataOutput = nil
|
||
audioDevice = nil
|
||
|
||
orientation.stop()
|
||
sink.unregister()
|
||
onEvent = nil
|
||
}
|
||
|
||
// MARK: - Camera flip
|
||
|
||
/// Replace the video input device (audio stays attached). Returns
|
||
/// the new previewSize. Must run on sessionQueue.
|
||
func setDescription(cameraId: String) throws -> CGSize {
|
||
precondition(!disposed)
|
||
try configureSession(forDeviceUniqueID: cameraId, replacing: true)
|
||
return previewSize
|
||
}
|
||
|
||
// MARK: - Flash + orientation
|
||
|
||
func setFlashMode(_ mode: AVCaptureDevice.FlashMode) {
|
||
flashMode = mode
|
||
}
|
||
|
||
func lockCaptureOrientation(_ next: DeviceOrientationFlutter) {
|
||
lockedOrientation = next
|
||
applyVideoOrientationOnPreview(next)
|
||
}
|
||
|
||
func unlockCaptureOrientation() {
|
||
lockedOrientation = nil
|
||
applyOrientationFollowDevice(orientation.current)
|
||
}
|
||
|
||
// MARK: - Photo
|
||
|
||
func takePicture(
|
||
snapshot: DeviceOrientationFlutter,
|
||
completion: @escaping (Result<String, NSError>) -> Void
|
||
) {
|
||
photoOutput.take(orientation: snapshot, flashMode: flashMode, completion: completion)
|
||
}
|
||
|
||
// MARK: - Video recording
|
||
|
||
/// Begin a recording. Must run on sessionQueue. Throws on writer
|
||
/// setup failure (typically a path / file-system issue).
|
||
/// [snapshot] is the orientation embedded as the file's track
|
||
/// transform — when the user is holding the device landscape,
|
||
/// pass landscape here and the file plays back landscape.
|
||
func startVideoRecording(
|
||
snapshot: DeviceOrientationFlutter
|
||
) throws {
|
||
precondition(!disposed)
|
||
guard videoRecorder == nil else {
|
||
throw NSError(
|
||
domain: "ux.camera",
|
||
code: -20,
|
||
userInfo: [
|
||
NSLocalizedDescriptionKey:
|
||
"Recording already in flight"
|
||
]
|
||
)
|
||
}
|
||
guard let videoOutput = videoDataOutput else {
|
||
throw NSError(
|
||
domain: "ux.camera",
|
||
code: -21,
|
||
userInfo: [NSLocalizedDescriptionKey: "Video output unavailable"]
|
||
)
|
||
}
|
||
|
||
let url = URL(fileURLWithPath: NSTemporaryDirectory())
|
||
.appendingPathComponent("ux_camera_\(UUID().uuidString).mp4")
|
||
|
||
// Audio is viable only when both the device input attached AND
|
||
// the audio output can recommend writer settings. Empty
|
||
// recommended settings means the audio path can't be muxed —
|
||
// telegram-ios fails the whole recording in that case
|
||
// (CameraOutput.swift:397-401); we silent-fall back to
|
||
// video-only so an audio-permission glitch doesn't break the
|
||
// page.
|
||
var audioViable = enableAudio
|
||
&& audioDeviceInput != nil
|
||
&& audioDataOutput != nil
|
||
|
||
let baseVideoSettings = videoOutput.recommendedVideoSettingsForAssetWriter(
|
||
writingTo: .mp4
|
||
) as? [String: Any]
|
||
|
||
var baseAudioSettings: [String: Any] = [:]
|
||
if audioViable, let ao = audioDataOutput {
|
||
baseAudioSettings = (ao.recommendedAudioSettingsForAssetWriter(
|
||
writingTo: .mp4
|
||
) as? [String: Any]) ?? [:]
|
||
if baseAudioSettings.isEmpty {
|
||
audioViable = false
|
||
}
|
||
}
|
||
|
||
let recorder = VideoRecorder(
|
||
url: url,
|
||
orientation: snapshot,
|
||
hasAudio: audioViable,
|
||
baseVideoSettings: baseVideoSettings,
|
||
baseAudioSettings: baseAudioSettings,
|
||
recorderQueue: recorderQueue
|
||
)
|
||
recorder.onDiagnostic = { [weak self] msg in
|
||
self?.emit(["event": "diagnostic", "message": msg])
|
||
}
|
||
try recorder.start()
|
||
|
||
videoRecorder = recorder
|
||
// Publish the recorder under the fanout's lock so the buffer
|
||
// queues see it on their next sample.
|
||
fanout.recorder = recorder
|
||
}
|
||
|
||
/// Stop the in-flight recording. Completion fires on
|
||
/// `recorderQueue` (which is `.async`'d here back to `.main` by
|
||
/// the plugin). Returns the file path or an error.
|
||
///
|
||
/// The fanout reference stays attached until `finishWriting`
|
||
/// completes — the recorder relies on *post-stop* sample buffers
|
||
/// crossing `recordingStopSampleTime` to trigger `maybeFinish`.
|
||
/// Detaching the feed at the wrong moment (before stop) is what
|
||
/// caused the 3-second watchdog to be the only thing finishing
|
||
/// the writer.
|
||
func stopVideoRecording(
|
||
completion: @escaping (Result<URL, NSError>) -> Void
|
||
) {
|
||
guard let recorder = videoRecorder else {
|
||
completion(.failure(NSError(
|
||
domain: "ux.camera",
|
||
code: -22,
|
||
userInfo: [NSLocalizedDescriptionKey: "No recording in flight"]
|
||
)))
|
||
return
|
||
}
|
||
|
||
recorder.stop { [weak self] outcome in
|
||
guard let self = self else {
|
||
completion(outcome)
|
||
return
|
||
}
|
||
self.sessionQueue.async {
|
||
self.fanout.recorder = nil
|
||
self.videoRecorder = nil
|
||
}
|
||
completion(outcome)
|
||
}
|
||
}
|
||
|
||
// MARK: - Private
|
||
|
||
private func configureSession(
|
||
forDeviceUniqueID cameraId: String,
|
||
replacing: Bool
|
||
) throws {
|
||
guard let device = AVCaptureDevice(uniqueID: cameraId) else {
|
||
throw NSError(
|
||
domain: "ux.camera",
|
||
code: -1,
|
||
userInfo: [NSLocalizedDescriptionKey: "Camera \(cameraId) not found"]
|
||
)
|
||
}
|
||
|
||
var caughtError: NSError?
|
||
|
||
session.configure {
|
||
if replacing, let oldInput = deviceInput {
|
||
session.av.removeInput(oldInput)
|
||
}
|
||
|
||
do {
|
||
let newInput = try AVCaptureDeviceInput(device: device)
|
||
guard session.av.canAddInput(newInput) else {
|
||
throw NSError(
|
||
domain: "ux.camera",
|
||
code: -1,
|
||
userInfo: [NSLocalizedDescriptionKey: "Cannot add input"]
|
||
)
|
||
}
|
||
session.av.addInput(newInput)
|
||
deviceInput = newInput
|
||
} catch let error as NSError {
|
||
caughtError = error
|
||
return
|
||
}
|
||
|
||
if !replacing {
|
||
if session.av.canSetSessionPreset(.high) {
|
||
session.av.sessionPreset = .high
|
||
}
|
||
|
||
let videoOutput = AVCaptureVideoDataOutput()
|
||
videoOutput.videoSettings = [
|
||
kCVPixelBufferPixelFormatTypeKey as String:
|
||
kCVPixelFormatType_32BGRA,
|
||
]
|
||
videoOutput.alwaysDiscardsLateVideoFrames = true
|
||
videoOutput.setSampleBufferDelegate(
|
||
fanout,
|
||
queue: videoBufferQueue
|
||
)
|
||
if session.av.canAddOutput(videoOutput) {
|
||
session.av.addOutput(videoOutput)
|
||
}
|
||
videoDataOutput = videoOutput
|
||
|
||
if session.av.canAddOutput(photoOutput.avOutput) {
|
||
session.av.addOutput(photoOutput.avOutput)
|
||
}
|
||
|
||
if enableAudio,
|
||
let mic = AVCaptureDevice.default(for: .audio) {
|
||
do {
|
||
let audioInput = try AVCaptureDeviceInput(device: mic)
|
||
if session.av.canAddInput(audioInput) {
|
||
session.av.addInput(audioInput)
|
||
audioDevice = mic
|
||
audioDeviceInput = audioInput
|
||
}
|
||
} catch {
|
||
// Don't fail the whole setup over audio — fall
|
||
// through; the recording will simply have no
|
||
// audio track.
|
||
}
|
||
|
||
let audioOutput = AVCaptureAudioDataOutput()
|
||
audioOutput.setSampleBufferDelegate(
|
||
fanout,
|
||
queue: audioBufferQueue
|
||
)
|
||
if session.av.canAddOutput(audioOutput) {
|
||
session.av.addOutput(audioOutput)
|
||
audioDataOutput = audioOutput
|
||
}
|
||
}
|
||
}
|
||
|
||
// Apply preview-output orientation. Mirroring is deliberately
|
||
// NOT set here — the data output feeds both the preview
|
||
// texture and the recorder, so mirroring at the connection
|
||
// would land in the recorded MP4 too. Telegram avoids this
|
||
// by mirroring at the preview-LAYER level (CALayer transform
|
||
// in `CameraPreviewView.mirroring`). Our FlutterTexture
|
||
// equivalent is a `Transform.flip` in [CameraThumb] for the
|
||
// front camera — raw sensor feed at capture, mirror as a
|
||
// playback decision.
|
||
if let videoConn = videoDataOutput?.connection(with: .video) {
|
||
videoConn.applyUxCaptureOrientation(
|
||
lockedOrientation ?? orientation.current
|
||
)
|
||
if videoConn.isVideoMirroringSupported {
|
||
videoConn.automaticallyAdjustsVideoMirroring = false
|
||
videoConn.isVideoMirrored = false
|
||
}
|
||
}
|
||
|
||
self.device = device
|
||
CaptureDevice.applyDefaults(device)
|
||
|
||
let dims = CMVideoFormatDescriptionGetDimensions(
|
||
device.activeFormat.formatDescription
|
||
)
|
||
previewSize = CGSize(width: CGFloat(dims.width), height: CGFloat(dims.height))
|
||
}
|
||
|
||
if let error = caughtError {
|
||
throw error
|
||
}
|
||
}
|
||
|
||
private func applyOrientationFollowDevice(_ next: DeviceOrientationFlutter) {
|
||
// When a lock is in effect the preview ignores physical
|
||
// rotation — the lock wins.
|
||
guard lockedOrientation == nil else { return }
|
||
applyVideoOrientationOnPreview(next)
|
||
}
|
||
|
||
private func applyVideoOrientationOnPreview(_ next: DeviceOrientationFlutter) {
|
||
videoDataOutput?.connection(with: .video)?
|
||
.applyUxCaptureOrientation(next)
|
||
}
|
||
|
||
private func emit(_ extras: [String: Any]) {
|
||
var payload: [String: Any] = ["handle": handle]
|
||
payload.merge(extras, uniquingKeysWith: { _, new in new })
|
||
DispatchQueue.main.async { [weak self] in self?.onEvent?(payload) }
|
||
}
|
||
}
|
||
|
||
/// Single sample-buffer delegate for both video + audio outputs.
|
||
/// Forwards video frames to [PreviewSink] and (when a recording is
|
||
/// active) both video and audio sample buffers to [VideoRecorder].
|
||
///
|
||
/// The `recorder` reference is cross-queue: written from
|
||
/// `sessionQueue` (set on startVideoRecording, cleared on
|
||
/// stopVideoRecording), read from `videoBufferQueue` and
|
||
/// `audioBufferQueue` (once per sample). An `NSLock` guards each
|
||
/// access — cheap, ~tens of nanoseconds per frame.
|
||
private final class SampleFanout: NSObject,
|
||
AVCaptureVideoDataOutputSampleBufferDelegate,
|
||
AVCaptureAudioDataOutputSampleBufferDelegate
|
||
{
|
||
private let sink: PreviewSink
|
||
private let recorderLock = NSLock()
|
||
private var _recorder: VideoRecorder?
|
||
|
||
var recorder: VideoRecorder? {
|
||
get {
|
||
recorderLock.lock(); defer { recorderLock.unlock() }
|
||
return _recorder
|
||
}
|
||
set {
|
||
recorderLock.lock(); defer { recorderLock.unlock() }
|
||
_recorder = newValue
|
||
}
|
||
}
|
||
|
||
init(sink: PreviewSink) {
|
||
self.sink = sink
|
||
}
|
||
|
||
func captureOutput(
|
||
_ output: AVCaptureOutput,
|
||
didOutput sampleBuffer: CMSampleBuffer,
|
||
from connection: AVCaptureConnection
|
||
) {
|
||
if output is AVCaptureVideoDataOutput {
|
||
sink.receive(sampleBuffer: sampleBuffer)
|
||
recorder?.appendVideo(sampleBuffer)
|
||
} else if output is AVCaptureAudioDataOutput {
|
||
recorder?.appendAudio(sampleBuffer)
|
||
}
|
||
}
|
||
}
|