Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 917b4fa

Browse files
cikelengfengcikelengfeng
committed
iOS rendering backend compatible with NV12(420v/420f) pixelbuffer
1 parent 626244a commit 917b4fa

4 files changed

Lines changed: 230 additions & 13 deletions

File tree

‎shell/platform/darwin/ios/ios_external_texture_gl.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ class IOSExternalTextureGL final : public Texture {
2525
fml::CFRef<CVOpenGLESTextureCacheRef> cache_ref_;
2626
fml::CFRef<CVOpenGLESTextureRef> texture_ref_;
2727
fml::CFRef<CVPixelBufferRef> buffer_ref_;
28+
OSType pixelFormat = 0;
29+
fml::CFRef<CVOpenGLESTextureRef> y_texture_ref_;
30+
fml::CFRef<CVOpenGLESTextureRef> uv_texture_ref_;
2831

2932
// |Texture|
3033
void Paint(SkCanvas& canvas,
@@ -51,6 +54,16 @@ class IOSExternalTextureGL final : public Texture {
5154

5255
bool NeedUpdateTexture(bool freeze);
5356

57+
bool IsTexturesAvailable();
58+
59+
void CreateYUVTexturesFromPixelBuffer();
60+
61+
void CreateRGBATextureFromPixelBuffer();
62+
63+
sk_sp<SkImage> CreateImageFromYUVTextures(GrContext* context, const SkRect& bounds);
64+
65+
sk_sp<SkImage> CreateImageFromRGBATexture(GrContext* context, const SkRect& bounds);
66+
5467
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureGL);
5568
};
5669

‎shell/platform/darwin/ios/ios_external_texture_gl.mm‎

Lines changed: 95 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010

1111
#include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
1212
#include "third_party/skia/include/core/SkSurface.h"
13+
#include "third_party/skia/include/core/SkYUVAIndex.h"
1314
#include "third_party/skia/include/gpu/GrBackendSurface.h"
15+
#include "third_party/skia/src/gpu/gl/GrGLDefines.h"
1416

1517
namespace flutter {
1618

@@ -41,6 +43,15 @@
4143
if (buffer_ref_ == nullptr) {
4244
return;
4345
}
46+
if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
47+
pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
48+
CreateYUVTexturesFromPixelBuffer();
49+
} else {
50+
CreateRGBATextureFromPixelBuffer();
51+
}
52+
}
53+
54+
void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() {
4455
CVOpenGLESTextureRef texture;
4556
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
4657
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_RGBA,
@@ -54,10 +65,82 @@
5465
}
5566
}
5667

68+
void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() {
69+
size_t width = CVPixelBufferGetWidth(buffer_ref_);
70+
size_t height = CVPixelBufferGetHeight(buffer_ref_);
71+
{
72+
CVOpenGLESTextureRef yTexture;
73+
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
74+
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_LUMINANCE, width,
75+
height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture);
76+
if (err != noErr) {
77+
FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err;
78+
} else {
79+
y_texture_ref_.Reset(yTexture);
80+
}
81+
}
82+
83+
{
84+
CVOpenGLESTextureRef uvTexture;
85+
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
86+
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA,
87+
width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &uvTexture);
88+
if (err != noErr) {
89+
FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err;
90+
} else {
91+
uv_texture_ref_.Reset(uvTexture);
92+
}
93+
}
94+
}
95+
96+
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromRGBATexture(GrContext* context,
97+
const SkRect& bounds) {
98+
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
99+
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
100+
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
101+
sk_sp<SkImage> image =
102+
SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
103+
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
104+
return image;
105+
}
106+
107+
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromYUVTextures(GrContext* context,
108+
const SkRect& bounds) {
109+
GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_),
110+
CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8};
111+
GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo);
112+
GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_),
113+
CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8};
114+
GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo,
115+
uvTextureInfo);
116+
GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture};
117+
SkYUVAIndex yuvaIndices[4] = {
118+
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
119+
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
120+
SkYUVAIndex{
121+
1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture,
122+
// normal NV12 data V should be taken from the green channel, but
123+
// currently only the uv texture created by GL_LUMINANCE_ALPHA
124+
// can be used, so the V value is taken from the alpha channel
125+
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
126+
SkISize size{yBackendTexture.width(), yBackendTexture.height()};
127+
sk_sp<SkImage> image =
128+
SkImage::MakeFromYUVATextures(context, kRec601_SkYUVColorSpace, nv12TextureHandles,
129+
yuvaIndices, size, kTopLeft_GrSurfaceOrigin, nullptr);
130+
return image;
131+
}
132+
133+
bool IOSExternalTextureGL::IsTexturesAvailable() {
134+
return ((pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
135+
pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) &&
136+
(y_texture_ref_ && uv_texture_ref_)) ||
137+
(pixelFormat == kCVPixelFormatType_32BGRA && texture_ref_);
138+
}
139+
57140
bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
58141
// Update texture if `texture_ref_` is reset to `nullptr` when GrContext
59142
// is destroyed or new frame is ready.
60-
return (!freeze && new_frame_ready_) || !texture_ref_;
143+
return (!freeze && new_frame_ready_) || !IsTexturesAvailable();
61144
}
62145

63146
void IOSExternalTextureGL::Paint(SkCanvas& canvas,
@@ -70,19 +153,23 @@
70153
auto pixelBuffer = [external_texture_.get() copyPixelBuffer];
71154
if (pixelBuffer) {
72155
buffer_ref_.Reset(pixelBuffer);
156+
pixelFormat = CVPixelBufferGetPixelFormatType(buffer_ref_);
73157
}
74158
CreateTextureFromPixelBuffer();
75159
new_frame_ready_ = false;
76160
}
77-
if (!texture_ref_) {
161+
if (!IsTexturesAvailable()) {
78162
return;
79163
}
80-
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
81-
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
82-
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
83-
sk_sp<SkImage> image =
84-
SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
85-
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
164+
165+
sk_sp<SkImage> image = nullptr;
166+
if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
167+
pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
168+
image = CreateImageFromYUVTextures(context, bounds);
169+
} else {
170+
image = CreateImageFromRGBATexture(context, bounds);
171+
}
172+
86173
FML_DCHECK(image) << "Failed to create SkImage from Texture.";
87174
if (image) {
88175
SkPaint paint;

‎shell/platform/darwin/ios/ios_external_texture_metal.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class IOSExternalTextureMetal final : public Texture {
3333
std::atomic_bool texture_frame_available_;
3434
fml::CFRef<CVPixelBufferRef> last_pixel_buffer_;
3535
sk_sp<SkImage> external_image_;
36+
OSType pixelFormat = 0;
3637

3738
// |Texture|
3839
void Paint(SkCanvas& canvas,
@@ -55,6 +56,10 @@ class IOSExternalTextureMetal final : public Texture {
5556

5657
sk_sp<SkImage> WrapExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
5758
GrContext* context) const;
59+
sk_sp<SkImage> WrapRGBAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
60+
GrContext* context) const;
61+
sk_sp<SkImage> WrapYUVAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
62+
GrContext* context) const;
5863

5964
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
6065
};

‎shell/platform/darwin/ios/ios_external_texture_metal.mm‎

Lines changed: 117 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
66

77
#include "flutter/fml/logging.h"
8+
#include "third_party/skia/include/core/SkYUVAIndex.h"
89
#include "third_party/skia/include/gpu/GrBackendSurface.h"
910
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
1011

@@ -34,6 +35,8 @@
3435
auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
3536
if (!pixel_buffer) {
3637
pixel_buffer = std::move(last_pixel_buffer_);
38+
} else {
39+
pixelFormat = CVPixelBufferGetPixelFormatType(pixel_buffer);
3740
}
3841

3942
// If the application told us there was a texture frame available but did not provide one when
@@ -64,9 +67,123 @@
6467
return nullptr;
6568
}
6669

70+
sk_sp<SkImage> image = nullptr;
71+
if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
72+
pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
73+
image = WrapYUVAExternalPixelBuffer(pixel_buffer, context);
74+
} else {
75+
image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
76+
}
77+
78+
if (!image) {
79+
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
80+
}
81+
82+
return image;
83+
}
84+
85+
sk_sp<SkImage> IOSExternalTextureMetal::WrapYUVAExternalPixelBuffer(
86+
fml::CFRef<CVPixelBufferRef> pixel_buffer,
87+
GrContext* context) const {
6788
auto texture_size =
6889
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
90+
CVMetalTextureRef y_metal_texture_raw = NULL;
91+
{
92+
auto cv_return =
93+
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
94+
texture_cache_, // texture cache
95+
pixel_buffer, // source image
96+
NULL, // texture attributes
97+
MTLPixelFormatR8Unorm, // pixel format
98+
texture_size.width(), // width
99+
texture_size.height(), // height
100+
0u, // plane index
101+
&y_metal_texture_raw // [out] texture
102+
);
69103

104+
if (cv_return != kCVReturnSuccess) {
105+
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
106+
return nullptr;
107+
}
108+
}
109+
110+
CVMetalTextureRef uv_metal_texture_raw = NULL;
111+
{
112+
auto cv_return =
113+
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
114+
texture_cache_, // texture cache
115+
pixel_buffer, // source image
116+
NULL, // texture attributes
117+
MTLPixelFormatRG8Unorm, // pixel format
118+
texture_size.width() / 2, // width
119+
texture_size.height() / 2, // height
120+
1u, // plane index
121+
&uv_metal_texture_raw // [out] texture
122+
);
123+
124+
if (cv_return != kCVReturnSuccess) {
125+
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
126+
return nullptr;
127+
}
128+
}
129+
130+
fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);
131+
132+
GrMtlTextureInfo y_skia_texture_info;
133+
y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
134+
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};
135+
136+
GrBackendTexture y_skia_backend_texture(texture_size.width(), // width
137+
texture_size.height(), // height
138+
GrMipMapped ::kNo, // mip-mapped
139+
y_skia_texture_info // texture info
140+
);
141+
142+
fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);
143+
144+
GrMtlTextureInfo uv_skia_texture_info;
145+
uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
146+
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};
147+
148+
GrBackendTexture uv_skia_backend_texture(texture_size.width(), // width
149+
texture_size.height(), // height
150+
GrMipMapped ::kNo, // mip-mapped
151+
uv_skia_texture_info // texture info
152+
);
153+
GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture};
154+
SkYUVAIndex yuvaIndices[4] = {
155+
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
156+
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
157+
SkYUVAIndex{1,
158+
SkColorChannel::kG}, // Read V data from the green channel of the second texture
159+
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
160+
161+
struct ImageCaptures {
162+
fml::CFRef<CVPixelBufferRef> buffer;
163+
fml::CFRef<CVMetalTextureRef> y_texture;
164+
fml::CFRef<CVMetalTextureRef> uv_texture;
165+
};
166+
167+
auto captures = std::make_unique<ImageCaptures>();
168+
captures->buffer = std::move(pixel_buffer);
169+
captures->y_texture = std::move(y_metal_texture);
170+
captures->uv_texture = std::move(uv_metal_texture);
171+
172+
SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
173+
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
174+
delete captures;
175+
};
176+
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
177+
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size,
178+
kTopLeft_GrSurfaceOrigin, nullptr, release_proc, captures.release());
179+
return image;
180+
}
181+
182+
sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
183+
fml::CFRef<CVPixelBufferRef> pixel_buffer,
184+
GrContext* context) const {
185+
auto texture_size =
186+
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
70187
CVMetalTextureRef metal_texture_raw = NULL;
71188
auto cv_return =
72189
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
@@ -121,11 +238,6 @@ GrBackendTexture skia_backend_texture(texture_size.width(), // width
121238
captures.release() // release context
122239

123240
);
124-
125-
if (!image) {
126-
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
127-
}
128-
129241
return image;
130242
}
131243

0 commit comments

Comments
 (0)