support External input video or audio ,set LFLiveCaptureTypeMask

Author: chenliming

LFLiveKit.xcodeproj/project.xcworkspace/xcuserdata/admin.xcuserdatad/UserInterfaceState.xcuserstate

@@ -15,6 +15,26 @@
 #import "LFLiveVideoConfiguration.h"
 #import "LFLiveDebug.h"
 
+typedef NS_ENUM(NSInteger,LFLiveCaptureType) {
+    LFLiveCaptureAudio,         //< capture only audio
+    LFLiveCaptureVideo,         //< capture onlt video
+    LFLiveInputAudio,           //< only audio (External input audio)
+    LFLiveInputVideo,           //< only video (External input video)
+};
+
+
+///< 用来控制采集类型（可以内部采集也可以外部传入等各种组合，支持单音频与单视频,外部输入适用于录屏，无人机等外设介入）
+typedef NS_ENUM(NSInteger,LFLiveCaptureTypeMask) {
+    LFLiveCaptureMaskAudio = (1 << LFLiveCaptureAudio),                                 ///< only inner capture audio (no video)
+    LFLiveCaptureMaskVideo = (1 << LFLiveCaptureVideo),                                 ///< only inner capture video (no audio)
+    LFLiveInputMaskAudio = (1 << LFLiveInputAudio),                                     ///< only outer input audio (no video)
+    LFLiveInputMaskVideo = (1 << LFLiveInputVideo),                                     ///< only outer input video (no audio)
+    LFLiveCaptureMaskAll = (LFLiveCaptureMaskAudio | LFLiveCaptureMaskVideo),           ///< inner capture audio and video
+    LFLiveInputMaskAll = (LFLiveInputMaskAudio | LFLiveInputMaskVideo),                 ///< outer input audio and video(method see pushVideo and pushAudio)
+    LFLiveCaptureMaskAudioInputVideo = (LFLiveCaptureMaskAudio | LFLiveInputMaskVideo), ///< inner capture audio and outer input video(method pushVideo and setRunning)
+    LFLiveCaptureMaskVideoInputAudio = (LFLiveCaptureMaskVideo | LFLiveInputMaskAudio), ///< inner capture video and outer input audio(method pushAudio and setRunning)
+    LFLiveCaptureDefaultMask = LFLiveCaptureMaskAll                                     ///< default is inner capture audio and video
+};
 
 @class LFLiveSession;
 @protocol LFLiveSessionDelegate <NSObject>
@@ -78,6 +98,9 @@
 /** The status of the stream .*/
 @property (nonatomic, assign, readonly) LFLiveState state;
 
+/** The captureType control inner or outer audio and video .*/
+@property (nonatomic, assign, readonly) LFLiveCaptureTypeMask captureType;
+
 /** The showDebugInfo control streamInfo and uploadInfo(1s) *.*/
 @property (nonatomic, assign) BOOL showDebugInfo;
 
@@ -105,12 +128,23 @@
  */
 - (nullable instancetype)initWithAudioConfiguration:(nullable LFLiveAudioConfiguration *)audioConfiguration videoConfiguration:(nullable LFLiveVideoConfiguration *)videoConfiguration NS_DESIGNATED_INITIALIZER;
 
+/**
+ The designated initializer. Multiple instances with the same configuration will make the
+ capture unstable.
+ */
+- (nullable instancetype)initWithAudioConfiguration:(nullable LFLiveAudioConfiguration *)audioConfiguration videoConfiguration:(nullable LFLiveVideoConfiguration *)videoConfiguration captureType:(LFLiveCaptureTypeMask)captureType NS_DESIGNATED_INITIALIZER;
+
 /** The start stream .*/
 - (void)startLive:(nonnull LFLiveStreamInfo *)streamInfo;
 
 /** The stop stream .*/
 - (void)stopLive;
 
+/** support outer input yuv or rgb video(set LFLiveCaptureTypeMask) .*/
+- (void)pushVideo:(CVPixelBufferRef)pixelBuffer;
+
+/** support outer input pcm audio(set LFLiveCaptureTypeMask) .*/
+- (void)pushAudio:(AudioBufferList)audioBufferList;
 
 @end
 

LFLiveKit/LFLiveSession.h

@@ -17,15 +17,12 @@
 #import "LFGPUImageBeautyFilter.h"
 #import "LFH264VideoEncoder.h"
 
-#define LFLiveReportKey @"com.youku.liveSessionReport"
 
 @interface LFLiveSession ()<LFAudioCaptureDelegate, LFVideoCaptureDelegate, LFAudioEncodingDelegate, LFVideoEncodingDelegate, LFStreamSocketDelegate>
-{
-    dispatch_semaphore_t _lock;
-}
-///音频配置
+
+/// 音频配置
 @property (nonatomic, strong) LFLiveAudioConfiguration *audioConfiguration;
-///视频配置
+/// 视频配置
 @property (nonatomic, strong) LFLiveVideoConfiguration *videoConfiguration;
 /// 声音采集
 @property (nonatomic, strong) LFAudioCapture *audioCaptureSource;
@@ -38,17 +35,21 @@ @interface LFLiveSession ()<LFAudioCaptureDelegate, LFVideoCaptureDelegate, LFAu
 /// 上传
 @property (nonatomic, strong) id<LFStreamSocket> socket;
 
+
 #pragma mark -- 内部标识
-/// 上报
-@property (nonatomic, copy) dispatch_block_t reportBlock;
-/// debugInfo
+/// 调试信息
 @property (nonatomic, strong) LFLiveDebug *debugInfo;
-/// streamInfo
+/// 流信息
 @property (nonatomic, strong) LFLiveStreamInfo *streamInfo;
-/// uploading
+/// 是否开始上传
 @property (nonatomic, assign) BOOL uploading;
-/// state
+/// 当前状态
 @property (nonatomic, assign, readwrite) LFLiveState state;
+/// 当前直播type
+@property (nonatomic, assign, readwrite) LFLiveCaptureTypeMask captureType;
+
+/// 时间戳锁
+@property (nonatomic, strong) dispatch_semaphore_t lock;
 
 @end
 
@@ -68,20 +69,25 @@ @implementation LFLiveSession
 
 #pragma mark -- LifeCycle
 - (instancetype)initWithAudioConfiguration:(LFLiveAudioConfiguration *)audioConfiguration videoConfiguration:(LFLiveVideoConfiguration *)videoConfiguration {
-    if (!audioConfiguration || !videoConfiguration) @throw [NSException exceptionWithName:@"LFLiveSession init error" reason:@"audioConfiguration or videoConfiguration is nil " userInfo:nil];
+    return [self initWithAudioConfiguration:audioConfiguration videoConfiguration:videoConfiguration captureType:LFLiveCaptureDefaultMask];
+}
+
+- (nullable instancetype)initWithAudioConfiguration:(nullable LFLiveAudioConfiguration *)audioConfiguration videoConfiguration:(nullable LFLiveVideoConfiguration *)videoConfiguration captureType:(LFLiveCaptureTypeMask)captureType{
+    if((captureType & LFLiveCaptureMaskAudio || captureType & LFLiveInputMaskAudio) && !audioConfiguration) @throw [NSException exceptionWithName:@"LFLiveSession init error" reason:@"audioConfiguration is nil " userInfo:nil];
+    if((captureType & LFLiveCaptureMaskVideo || captureType & LFLiveInputMaskVideo) && !videoConfiguration) @throw [NSException exceptionWithName:@"LFLiveSession init error" reason:@"videoConfiguration is nil " userInfo:nil];
     if (self = [super init]) {
         _audioConfiguration = audioConfiguration;
         _videoConfiguration = videoConfiguration;
-        _lock = dispatch_semaphore_create(1);
         _adaptiveBitrate = NO;
         _isFirstFrame = YES;
+        _captureType = captureType;
     }
     return self;
 }
 
 - (void)dealloc {
-    self.audioCaptureSource.running = NO;
     self.videoCaptureSource.running = NO;
+    self.audioCaptureSource.running = NO;
 }
 
 #pragma mark -- CustomMethod
@@ -90,6 +96,7 @@ - (void)startLive:(LFLiveStreamInfo *)streamInfo {
     _streamInfo = streamInfo;
     _streamInfo.videoConfiguration = _videoConfiguration;
     _streamInfo.audioConfiguration = _audioConfiguration;
+    _streamInfo.needDropFrame = (self.captureType & LFLiveCaptureMaskVideo || self.captureType & LFLiveInputMaskVideo) ? YES : NO;//< 有视频执行丢帧算法
     [self.socket start];
 }
 
@@ -99,12 +106,24 @@ - (void)stopLive {
     self.socket = nil;
 }
 
+- (void)pushVideo:(CVPixelBufferRef)pixelBuffer{
+    if(self.captureType & LFLiveInputMaskVideo){
+        if (self.uploading) [self.videoEncoder encodeVideoData:pixelBuffer timeStamp:self.currentTimestamp];
+    }
+}
+
+- (void)pushAudio:(AudioBufferList)audioBufferList{
+    if(self.captureType & LFLiveInputMaskAudio){
+        if (self.uploading) [self.audioEncoder encodeAudioData:audioBufferList timeStamp:self.currentTimestamp];
+    }
+}
+
 #pragma mark -- CaptureDelegate
 - (void)captureOutput:(nullable LFAudioCapture *)capture audioBuffer:(AudioBufferList)inBufferList {
     if (self.uploading) [self.audioEncoder encodeAudioData:inBufferList timeStamp:self.currentTimestamp];
 }
 
-- (void)captureOutput:(nullable LFVideoCapture *)capture pixelBuffer:(nullable CVImageBufferRef)pixelBuffer {
+- (void)captureOutput:(nullable LFVideoCapture *)capture pixelBuffer:(nullable CVPixelBufferRef)pixelBuffer {
     if (self.uploading) [self.videoEncoder encodeVideoData:pixelBuffer timeStamp:self.currentTimestamp];
 }
 
@@ -156,18 +175,18 @@ - (void)socketDebug:(nullable id<LFStreamSocket>)socket debugInfo:(nullable LFLi
 }
 
 - (void)socketBufferStatus:(nullable id<LFStreamSocket>)socket status:(LFLiveBuffferState)status {
-    if (self.adaptiveBitrate) {
-        NSUInteger videoBitRate = [_videoEncoder videoBitRate];
+    if((self.captureType & LFLiveCaptureMaskVideo || self.captureType & LFLiveInputMaskVideo) && self.adaptiveBitrate){
+        NSUInteger videoBitRate = [self.videoEncoder videoBitRate];
         if (status == LFLiveBuffferDecline) {
             if (videoBitRate < _videoConfiguration.videoMaxBitRate) {
                 videoBitRate = videoBitRate + 50 * 1000;
-                [_videoEncoder setVideoBitRate:videoBitRate];
+                [self.videoEncoder setVideoBitRate:videoBitRate];
                 NSLog(@"Increase bitrate %@", @(videoBitRate));
             }
         } else {
-            if (videoBitRate > _videoConfiguration.videoMinBitRate) {
+            if (videoBitRate > self.videoConfiguration.videoMinBitRate) {
                 videoBitRate = videoBitRate - 100 * 1000;
-                [_videoEncoder setVideoBitRate:videoBitRate];
+                [self.videoEncoder setVideoBitRate:videoBitRate];
                 NSLog(@"Decline bitrate %@", @(videoBitRate));
             }
         }
@@ -284,16 +303,20 @@ - (UIView*)warterMarkView{
 
 - (LFAudioCapture *)audioCaptureSource {
     if (!_audioCaptureSource) {
-        _audioCaptureSource = [[LFAudioCapture alloc] initWithAudioConfiguration:_audioConfiguration];
-        _audioCaptureSource.delegate = self;
+        if(self.captureType & LFLiveCaptureMaskAudio){
+            _audioCaptureSource = [[LFAudioCapture alloc] initWithAudioConfiguration:_audioConfiguration];
+            _audioCaptureSource.delegate = self;
+        }
     }
     return _audioCaptureSource;
 }
 
 - (LFVideoCapture *)videoCaptureSource {
     if (!_videoCaptureSource) {
-        _videoCaptureSource = [[LFVideoCapture alloc] initWithVideoConfiguration:_videoConfiguration];
-        _videoCaptureSource.delegate = self;
+        if(self.captureType & LFLiveCaptureMaskVideo){
+            _videoCaptureSource = [[LFVideoCapture alloc] initWithVideoConfiguration:_videoConfiguration];
+            _videoCaptureSource.delegate = self;
+        }
     }
     return _videoCaptureSource;
 }
@@ -320,7 +343,7 @@ - (LFVideoCapture *)videoCaptureSource {
 
 - (id<LFStreamSocket>)socket {
     if (!_socket) {
-        _socket = [[LFStreamRTMPSocket alloc] initWithStream:self.streamInfo videoSize:self.videoConfiguration.videoSize reconnectInterval:self.reconnectInterval reconnectCount:self.reconnectCount];
+        _socket = [[LFStreamRTMPSocket alloc] initWithStream:self.streamInfo reconnectInterval:self.reconnectInterval reconnectCount:self.reconnectCount];
         [_socket setDelegate:self];
     }
     return _socket;
@@ -333,8 +356,15 @@ - (LFLiveStreamInfo *)streamInfo {
     return _streamInfo;
 }
 
+- (dispatch_semaphore_t)lock{
+    if(!_lock){
+        _lock = dispatch_semaphore_create(1);
+    }
+    return _lock;
+}
+
 - (uint64_t)currentTimestamp {
-    dispatch_semaphore_wait(_lock, DISPATCH_TIME_FOREVER);
+    dispatch_semaphore_wait(self.lock, DISPATCH_TIME_FOREVER);
     uint64_t currentts = 0;
     if (_isFirstFrame) {
         _timestamp = NOW;
@@ -343,7 +373,7 @@ - (uint64_t)currentTimestamp {
     } else {
         currentts = NOW - _timestamp;
     }
-    dispatch_semaphore_signal(_lock);
+    dispatch_semaphore_signal(self.lock);
     return currentts;
 }
 

LFLiveKit/LFLiveSession.m

@@ -13,7 +13,7 @@
 @class LFVideoCapture;
 /** LFVideoCapture callback videoData */
 @protocol LFVideoCaptureDelegate <NSObject>
-- (void)captureOutput:(nullable LFVideoCapture *)capture pixelBuffer:(nullable CVImageBufferRef)pixelBuffer;
+- (void)captureOutput:(nullable LFVideoCapture *)capture pixelBuffer:(nullable CVPixelBufferRef)pixelBuffer;
 @end
 
 @interface LFVideoCapture : NSObject

LFLiveKit/capture/LFVideoCapture.h

@@ -233,7 +233,7 @@ - (GPUImageAlphaBlendFilter *)blendFilter{
 - (UIView *)waterMarkContentView{
     if(!_waterMarkContentView){
         _waterMarkContentView = [UIView new];
-        _waterMarkContentView.frame = CGRectMake(0, 0, self.gpuImageView.frame.size.width, self.gpuImageView.frame.size.height);
+        _waterMarkContentView.frame = CGRectMake(0, 0, self.configuration.videoSize.width, self.configuration.videoSize.height);
         _waterMarkContentView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
     }
     return _waterMarkContentView;
@@ -254,14 +254,6 @@ - (void)processVideo:(GPUImageOutput *)output {
     @autoreleasepool {
         GPUImageFramebuffer *imageFramebuffer = output.framebufferForOutput;
         CVPixelBufferRef pixelBuffer = [imageFramebuffer pixelBuffer];
-        if(!CGSizeEqualToSize(_self.configuration.videoSize, imageFramebuffer.size)){
-            NSInteger width = ceil(imageFramebuffer.size.width);
-            NSInteger height = ceil(imageFramebuffer.size.height);
-            if(width %2 != 0) width = width + 1;
-            if(height %2 != 0) height = height + 1;
-            _self.configuration.videoSize = CGSizeMake(width, height);
-            _self.waterMarkContentView.frame = CGRectMake(0, 0,_self.configuration.videoSize.width, _self.configuration.videoSize.height);
-        }
         if (pixelBuffer && _self.delegate && [_self.delegate respondsToSelector:@selector(captureOutput:pixelBuffer:)]) {
             [_self.delegate captureOutput:_self pixelBuffer:pixelBuffer];
         }
@@ -309,18 +301,10 @@ - (void)reloadFilter{
         [self.output addTarget:self.gpuImageView];
     }
 
-    //< 输出大小自适应
-    if(self.configuration.videoSizeRespectingAspectRatio){
-        [self.filter forceProcessingAtSizeRespectingAspectRatio:self.configuration.videoSize];
-        [self.output forceProcessingAtSizeRespectingAspectRatio:self.configuration.videoSize];
-        [self.blendFilter forceProcessingAtSizeRespectingAspectRatio:self.configuration.videoSize];
-        [self.uiElementInput forceProcessingAtSizeRespectingAspectRatio:self.configuration.videoSize];
-    }else{
-        [self.filter forceProcessingAtSize:self.configuration.videoSize];
-        [self.output forceProcessingAtSize:self.configuration.videoSize];
-        [self.blendFilter forceProcessingAtSize:self.configuration.videoSize];
-        [self.uiElementInput forceProcessingAtSize:self.configuration.videoSize];
-    }
+    [self.filter forceProcessingAtSize:self.configuration.videoSize];
+    [self.output forceProcessingAtSize:self.configuration.videoSize];
+    [self.blendFilter forceProcessingAtSize:self.configuration.videoSize];
+    [self.uiElementInput forceProcessingAtSize:self.configuration.videoSize];
 
     //< 输出数据
     __weak typeof(self) _self = self;

LFLiveKit/capture/LFVideoCapture.m

@@ -98,7 +98,7 @@ - (void)dealloc {
 }
 
 #pragma mark -- LFVideoEncoder
-- (void)encodeVideoData:(CVImageBufferRef)pixelBuffer timeStamp:(uint64_t)timeStamp {
+- (void)encodeVideoData:(CVPixelBufferRef)pixelBuffer timeStamp:(uint64_t)timeStamp {
     if (_isBackGround) return;
 
     frameCount++;

LFLiveKit/coder/LFHardwareVideoEncoder.m

@@ -20,7 +20,7 @@
 /// 编码器抽象的接口
 @protocol LFVideoEncoding <NSObject>
 @required
-- (void)encodeVideoData:(nullable CVImageBufferRef)pixelBuffer timeStamp:(uint64_t)timeStamp;
+- (void)encodeVideoData:(nullable CVPixelBufferRef)pixelBuffer timeStamp:(uint64_t)timeStamp;
 - (void)stopEncoder;
 @optional
 @property (nonatomic, assign) NSInteger videoBitRate;