Open Radar

 
Summary:

When using AVVideoMaxKeyFrameIntervalDurationKey in conjunction with AVVideoAverageBitRateKey, the resulting bitrate drops as keyframe interval decreases.

This is unlike kVTCompressionPropertyKey_DataRateLimits, which holds the same bitrate regardless of keyframe interval setting. Use of the latter is undesirable as it omits B-frames.

Steps to Reproduce:
See attached code.

Bitrate is verified by the following ffprobe command:

ffprobe -v error $file -select_streams v:0 -show_entries stream=bit_rate -of default=noprint_wrappers=1

Expected Results:
Average bitrate is maintained.

Actual Results:
Average bitrate is missed.

Version/Build:
iOS 12.3

Configuration:
N/A

-----

- (void)test {
    NSArray *const bitRates = @[
                                @(2e6),
                                @(3e6),
                                @(4e6),
                                @(5e6),
                                @(7e6),
                                @(9e6),
                                @(12e6),
                                @(15e6)
                                ];

    NSArray *const urls = @[
                            [[NSBundle bundleForClass:[self class]] URLForResource:@"file1" withExtension:@"mp4"],
                            [[NSBundle bundleForClass:[self class]] URLForResource:@"file2" withExtension:@"mp4"],
                            [[NSBundle bundleForClass:[self class]] URLForResource:@"file3" withExtension:@"mp4"],
                            ];

    NSArray *const keyFrameIntervals = @[
                                         @(0.f),
                                         @(1.f),
                                         @(2.f),
                                         @(3.f),
                                         @(4.f),
                                         @(5.f),
                                         ];

    for (NSURL *url in urls) {
        for (NSNumber *bitRate in bitRates) {
            for (NSNumber *keyFrameInterval in keyFrameIntervals) {
                for (NSNumber *drl in @[@1, @2, @3]) {
                    [self testUrl:url bitRate:[bitRate floatValue] keyFrameInterval:[keyFrameInterval integerValue] drl:[drl integerValue]];
                }
            }
        }
    }
}

- (void)testUrl:(NSURL *)url bitRate:(CGFloat)bitRate keyFrameInterval:(NSInteger)keyFrameInterval drl:(NSInteger)drl {

    NSString *extension = nil;
    if (drl == 1) {
        extension = @"avg";
    } else if (drl == 2) {
        extension = @"drl";
    } else if (drl == 3) {
        extension = @"both";
    }

    NSString *filename = [NSString stringWithFormat:@"test-%@-%ld-bps-%ld-s-%@",
                          [[url URLByDeletingPathExtension] lastPathComponent],
                          (long)bitRate,
                          (long)keyFrameInterval,
                          extension];

    NSURL *const outputUrl = [[[NSURL fileURLWithPath:NSTemporaryDirectory()]
                               URLByAppendingPathComponent:filename]
                              URLByAppendingPathExtension:@"mp4"];

    if ([[NSFileManager defaultManager]fileExistsAtPath:outputUrl.path]) {
        [[NSFileManager defaultManager] removeItemAtURL:outputUrl error:nil];
    }

    AVAsset *const asset = [AVAsset assetWithURL:url];
    AVAssetTrack *const videoTrack = [[asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
    NSError *error = nil;
    AVAssetReader *const assetReader = [AVAssetReader assetReaderWithAsset:asset error:&error];
    XCTAssertNil(error);
    AVAssetWriter *const assetWriter = [AVAssetWriter assetWriterWithURL:outputUrl fileType:AVFileTypeMPEG4 error:&error];
    XCTAssertNil(error);

    NSMutableDictionary *const compressionProperties = [NSMutableDictionary new];
    compressionProperties[AVVideoProfileLevelKey] = AVVideoProfileLevelH264High41;
    compressionProperties[AVVideoMaxKeyFrameIntervalDurationKey] = @(keyFrameInterval);

    if (drl == 1 || drl == 3) {
        compressionProperties[AVVideoAverageBitRateKey] = @(bitRate);
    } else if (drl == 2 || drl == 3) {
        compressionProperties[(id)kVTCompressionPropertyKey_DataRateLimits] = @[@(bitRate / 8.f), @1];
    }

    AVAssetWriterInput *const input = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo
                                                                     outputSettings:@{
                                                                                      AVVideoCodecKey: AVVideoCodecTypeH264,
                                                                                      AVVideoWidthKey: @(videoTrack.naturalSize.width),
                                                                                      AVVideoHeightKey: @(videoTrack.naturalSize.height),
                                                                                      AVVideoCompressionPropertiesKey: compressionProperties
                                                                                      }];

    AVAssetReaderOutput *const output = [[AVAssetReaderTrackOutput alloc] initWithTrack:videoTrack
                                                                         outputSettings:@{
                                                                                          (id)kCVPixelBufferPixelFormatTypeKey:
                                                                                              @(kCVPixelFormatType_32BGRA),
                                                                                          }];

    [assetReader addOutput:output];
    [assetWriter addInput:input];

    XCTAssertNil(assetReader.error);
    XCTAssertNil(assetWriter.error);

    [assetReader startReading];
    [assetWriter startWriting];

    [assetWriter startSessionAtSourceTime:kCMTimeZero];

    const dispatch_queue_t queue = dispatch_queue_create("transcode", DISPATCH_QUEUE_SERIAL);
    const dispatch_semaphore_t transcode = dispatch_semaphore_create(0);
    [input requestMediaDataWhenReadyOnQueue:queue usingBlock:^{
        while ([input isReadyForMoreMediaData]) {
            const CMSampleBufferRef sampleBuffer = [output copyNextSampleBuffer];

            XCTAssertNil(assetReader.error);
            XCTAssertNil(assetWriter.error);

            if (sampleBuffer) {
                [input appendSampleBuffer:sampleBuffer];
                CFRelease(sampleBuffer);
            } else {
                [input markAsFinished];
                dispatch_semaphore_signal(transcode);
            }
        }
    }];

    dispatch_semaphore_wait(transcode, DISPATCH_TIME_FOREVER);

    const dispatch_semaphore_t write = dispatch_semaphore_create(0);
    [assetWriter finishWritingWithCompletionHandler:^{
        dispatch_semaphore_signal(write);
    }];

    XCTAssertNil(assetWriter.error);

    dispatch_semaphore_wait(write, DISPATCH_TIME_FOREVER);

    NSLog(@"Wrote to %@", outputUrl);
}