Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 6cb5d6c

Browse files
cikelengfengcikelengfeng
committed
iOS external texture supports rendering NV12 pixelbuffer
1 parent 36c5f60 commit 6cb5d6c

4 files changed

Lines changed: 248 additions & 40 deletions

File tree

‎shell/platform/darwin/ios/ios_external_texture_gl.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ class IOSExternalTextureGL final : public Texture {
2525
fml::CFRef<CVOpenGLESTextureCacheRef> cache_ref_;
2626
fml::CFRef<CVOpenGLESTextureRef> texture_ref_;
2727
fml::CFRef<CVPixelBufferRef> buffer_ref_;
28+
OSType pixel_format_ = 0;
29+
fml::CFRef<CVOpenGLESTextureRef> y_texture_ref_;
30+
fml::CFRef<CVOpenGLESTextureRef> uv_texture_ref_;
2831

2932
// |Texture|
3033
void Paint(SkCanvas& canvas,
@@ -51,6 +54,16 @@ class IOSExternalTextureGL final : public Texture {
5154

5255
bool NeedUpdateTexture(bool freeze);
5356

57+
bool IsTexturesAvailable() const;
58+
59+
void CreateYUVTexturesFromPixelBuffer();
60+
61+
void CreateRGBATextureFromPixelBuffer();
62+
63+
sk_sp<SkImage> CreateImageFromYUVTextures(GrContext* context, const SkRect& bounds);
64+
65+
sk_sp<SkImage> CreateImageFromRGBATexture(GrContext* context, const SkRect& bounds);
66+
5467
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureGL);
5568
};
5669

‎shell/platform/darwin/ios/ios_external_texture_gl.mm‎

Lines changed: 98 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010

1111
#include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
1212
#include "third_party/skia/include/core/SkSurface.h"
13+
#include "third_party/skia/include/core/SkYUVAIndex.h"
1314
#include "third_party/skia/include/gpu/GrBackendSurface.h"
1415
#include "third_party/skia/include/gpu/GrDirectContext.h"
16+
#include "third_party/skia/src/gpu/gl/GrGLDefines.h"
1517

1618
namespace flutter {
1719

@@ -42,10 +44,19 @@
4244
if (buffer_ref_ == nullptr) {
4345
return;
4446
}
47+
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
48+
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
49+
CreateYUVTexturesFromPixelBuffer();
50+
} else {
51+
CreateRGBATextureFromPixelBuffer();
52+
}
53+
}
54+
55+
void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() {
4556
CVOpenGLESTextureRef texture;
4657
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
47-
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_RGBA,
48-
static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
58+
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
59+
GL_RGBA, static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
4960
static_cast<int>(CVPixelBufferGetHeight(buffer_ref_)), GL_BGRA, GL_UNSIGNED_BYTE, 0,
5061
&texture);
5162
if (err != noErr) {
@@ -55,10 +66,83 @@
5566
}
5667
}
5768

69+
void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() {
70+
size_t width = CVPixelBufferGetWidth(buffer_ref_);
71+
size_t height = CVPixelBufferGetHeight(buffer_ref_);
72+
{
73+
CVOpenGLESTextureRef yTexture;
74+
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
75+
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
76+
GL_LUMINANCE, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture);
77+
if (err != noErr) {
78+
FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err;
79+
} else {
80+
y_texture_ref_.Reset(yTexture);
81+
}
82+
}
83+
84+
{
85+
CVOpenGLESTextureRef uvTexture;
86+
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
87+
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
88+
GL_LUMINANCE_ALPHA, width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1,
89+
&uvTexture);
90+
if (err != noErr) {
91+
FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err;
92+
} else {
93+
uv_texture_ref_.Reset(uvTexture);
94+
}
95+
}
96+
}
97+
98+
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromRGBATexture(GrContext* context,
99+
const SkRect& bounds) {
100+
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
101+
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
102+
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
103+
sk_sp<SkImage> image = SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
104+
kRGBA_8888_SkColorType, kPremul_SkAlphaType,
105+
/*imageColorSpace=*/nullptr);
106+
return image;
107+
}
108+
109+
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromYUVTextures(GrContext* context,
110+
const SkRect& bounds) {
111+
GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_),
112+
CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8};
113+
GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo);
114+
GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_),
115+
CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8};
116+
GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo,
117+
uvTextureInfo);
118+
GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture};
119+
SkYUVAIndex yuvaIndices[4] = {
120+
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
121+
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
122+
SkYUVAIndex{
123+
1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture,
124+
// normal NV12 data V should be taken from the green channel, but
125+
// currently only the uv texture created by GL_LUMINANCE_ALPHA
126+
// can be used, so the V value is taken from the alpha channel
127+
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
128+
SkISize size{yBackendTexture.width(), yBackendTexture.height()};
129+
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
130+
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, size,
131+
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr);
132+
return image;
133+
}
134+
135+
bool IOSExternalTextureGL::IsTexturesAvailable() const {
136+
return ((pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
137+
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) &&
138+
(y_texture_ref_ && uv_texture_ref_)) ||
139+
(pixel_format_ == kCVPixelFormatType_32BGRA && texture_ref_);
140+
}
141+
58142
bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
59143
// Update texture if `texture_ref_` is reset to `nullptr` when GrContext
60144
// is destroyed or new frame is ready.
61-
return (!freeze && new_frame_ready_) || !texture_ref_;
145+
return (!freeze && new_frame_ready_) || !IsTexturesAvailable();
62146
}
63147

64148
void IOSExternalTextureGL::Paint(SkCanvas& canvas,
@@ -71,19 +155,23 @@
71155
auto pixelBuffer = [external_texture_.get() copyPixelBuffer];
72156
if (pixelBuffer) {
73157
buffer_ref_.Reset(pixelBuffer);
158+
pixel_format_ = CVPixelBufferGetPixelFormatType(buffer_ref_);
74159
}
75160
CreateTextureFromPixelBuffer();
76161
new_frame_ready_ = false;
77162
}
78-
if (!texture_ref_) {
163+
if (!IsTexturesAvailable()) {
79164
return;
80165
}
81-
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
82-
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
83-
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
84-
sk_sp<SkImage> image =
85-
SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
86-
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
166+
167+
sk_sp<SkImage> image = nullptr;
168+
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
169+
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
170+
image = CreateImageFromYUVTextures(context, bounds);
171+
} else {
172+
image = CreateImageFromRGBATexture(context, bounds);
173+
}
174+
87175
FML_DCHECK(image) << "Failed to create SkImage from Texture.";
88176
if (image) {
89177
SkPaint paint;

‎shell/platform/darwin/ios/ios_external_texture_metal.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class IOSExternalTextureMetal final : public Texture {
3333
std::atomic_bool texture_frame_available_;
3434
fml::CFRef<CVPixelBufferRef> last_pixel_buffer_;
3535
sk_sp<SkImage> external_image_;
36+
OSType pixel_format_ = 0;
3637

3738
// |Texture|
3839
void Paint(SkCanvas& canvas,
@@ -55,6 +56,10 @@ class IOSExternalTextureMetal final : public Texture {
5556

5657
sk_sp<SkImage> WrapExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
5758
GrDirectContext* context) const;
59+
sk_sp<SkImage> WrapRGBAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
60+
GrDirectContext* context) const;
61+
sk_sp<SkImage> WrapNV12ExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
62+
GrDirectContext* context) const;
5863

5964
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
6065
};

‎shell/platform/darwin/ios/ios_external_texture_metal.mm‎

Lines changed: 132 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
66

77
#include "flutter/fml/logging.h"
8+
#include "third_party/skia/include/core/SkYUVAIndex.h"
89
#include "third_party/skia/include/gpu/GrBackendSurface.h"
910
#include "third_party/skia/include/gpu/GrDirectContext.h"
1011
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
@@ -35,6 +36,8 @@
3536
auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
3637
if (!pixel_buffer) {
3738
pixel_buffer = std::move(last_pixel_buffer_);
39+
} else {
40+
pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer);
3841
}
3942

4043
// If the application told us there was a texture frame available but did not provide one when
@@ -65,21 +68,130 @@
6568
return nullptr;
6669
}
6770

71+
sk_sp<SkImage> image = nullptr;
72+
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
73+
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
74+
image = WrapNV12ExternalPixelBuffer(pixel_buffer, context);
75+
} else {
76+
image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
77+
}
78+
79+
if (!image) {
80+
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
81+
}
82+
83+
return image;
84+
}
85+
86+
sk_sp<SkImage> IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer(
87+
fml::CFRef<CVPixelBufferRef> pixel_buffer,
88+
GrDirectContext* context) const {
6889
auto texture_size =
6990
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
91+
CVMetalTextureRef y_metal_texture_raw = nullptr;
92+
{
93+
auto cv_return =
94+
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
95+
/*textureCache=*/texture_cache_,
96+
/*sourceImage=*/pixel_buffer,
97+
/*textureAttributes=*/nullptr,
98+
/*pixelFormat=*/MTLPixelFormatR8Unorm,
99+
/*width=*/texture_size.width(),
100+
/*height=*/texture_size.height(),
101+
/*planeIndex=*/0u,
102+
/*texture=*/&y_metal_texture_raw);
103+
104+
if (cv_return != kCVReturnSuccess) {
105+
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
106+
return nullptr;
107+
}
108+
}
109+
110+
CVMetalTextureRef uv_metal_texture_raw = nullptr;
111+
{
112+
auto cv_return =
113+
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
114+
/*textureCache=*/texture_cache_,
115+
/*sourceImage=*/pixel_buffer,
116+
/*textureAttributes=*/nullptr,
117+
/*pixelFormat=*/MTLPixelFormatRG8Unorm,
118+
/*width=*/texture_size.width() / 2,
119+
/*height=*/texture_size.height() / 2,
120+
/*planeIndex=*/1u,
121+
/*texture=*/&uv_metal_texture_raw);
122+
123+
if (cv_return != kCVReturnSuccess) {
124+
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
125+
return nullptr;
126+
}
127+
}
128+
129+
fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);
130+
131+
GrMtlTextureInfo y_skia_texture_info;
132+
y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
133+
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};
70134

71-
CVMetalTextureRef metal_texture_raw = NULL;
135+
GrBackendTexture y_skia_backend_texture(/*width=*/texture_size.width(),
136+
/*height=*/texture_size.height(),
137+
/*mipMapped=*/GrMipMapped ::kNo,
138+
/*textureInfo=*/y_skia_texture_info);
139+
140+
fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);
141+
142+
GrMtlTextureInfo uv_skia_texture_info;
143+
uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
144+
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};
145+
146+
GrBackendTexture uv_skia_backend_texture(/*width=*/texture_size.width(),
147+
/*height=*/texture_size.height(),
148+
/*mipMapped=*/GrMipMapped ::kNo,
149+
/*textureInfo=*/uv_skia_texture_info);
150+
GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture};
151+
SkYUVAIndex yuvaIndices[4] = {
152+
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
153+
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
154+
SkYUVAIndex{1,
155+
SkColorChannel::kG}, // Read V data from the green channel of the second texture
156+
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
157+
158+
struct ImageCaptures {
159+
fml::CFRef<CVPixelBufferRef> buffer;
160+
fml::CFRef<CVMetalTextureRef> y_texture;
161+
fml::CFRef<CVMetalTextureRef> uv_texture;
162+
};
163+
164+
auto captures = std::make_unique<ImageCaptures>();
165+
captures->buffer = std::move(pixel_buffer);
166+
captures->y_texture = std::move(y_metal_texture);
167+
captures->uv_texture = std::move(uv_metal_texture);
168+
169+
SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
170+
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
171+
delete captures;
172+
};
173+
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
174+
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size,
175+
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr, release_proc, captures.release());
176+
return image;
177+
}
178+
179+
sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
180+
fml::CFRef<CVPixelBufferRef> pixel_buffer,
181+
GrDirectContext* context) const {
182+
auto texture_size =
183+
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
184+
CVMetalTextureRef metal_texture_raw = nullptr;
72185
auto cv_return =
73-
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
74-
texture_cache_, // texture cache
75-
pixel_buffer, // source image
76-
NULL, // texture attributes
77-
MTLPixelFormatBGRA8Unorm, // pixel format
78-
texture_size.width(), // width
79-
texture_size.height(), // height
80-
0u, // plane index
81-
&metal_texture_raw // [out] texture
82-
);
186+
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
187+
/*textureCache=*/texture_cache_,
188+
/*sourceImage=*/pixel_buffer,
189+
/*textureAttributes=*/nullptr,
190+
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
191+
/*width=*/texture_size.width(),
192+
/*height=*/texture_size.height(),
193+
/*planeIndex=*/0u,
194+
/*texture=*/&metal_texture_raw);
83195

84196
if (cv_return != kCVReturnSuccess) {
85197
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
@@ -92,11 +204,10 @@
92204
skia_texture_info.fTexture = sk_cf_obj<const void*>{
93205
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};
94206

95-
GrBackendTexture skia_backend_texture(texture_size.width(), // width
96-
texture_size.height(), // height
97-
GrMipMapped ::kNo, // mip-mapped
98-
skia_texture_info // texture info
99-
);
207+
GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(),
208+
/*height=*/texture_size.height(),
209+
/*mipMapped=*/GrMipMapped ::kNo,
210+
/*textureInfo=*/skia_texture_info);
100211

101212
struct ImageCaptures {
102213
fml::CFRef<CVPixelBufferRef> buffer;
@@ -112,21 +223,12 @@ GrBackendTexture skia_backend_texture(texture_size.width(), // width
112223
delete captures;
113224
};
114225

115-
auto image = SkImage::MakeFromTexture(context, // context
116-
skia_backend_texture, // backend texture
117-
kTopLeft_GrSurfaceOrigin, // origin
118-
kBGRA_8888_SkColorType, // color type
119-
kPremul_SkAlphaType, // alpha type
120-
nullptr, // color space
121-
release_proc, // release proc
122-
captures.release() // release context
123-
124-
);
125-
126-
if (!image) {
127-
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
128-
}
226+
auto image =
227+
SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin,
228+
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
229+
/*imageColorSpace=*/nullptr, release_proc, captures.release()
129230

231+
);
130232
return image;
131233
}
132234

0 commit comments

Comments
 (0)