app,gpu,op: [android,ios,macos,windows] add ExternalOp paint for external views

This change adds paint.ExternalOp, which allows native views, activities, or HWNDs to appear inside a Gio layout.

The implementation uses a punch-through method. Gio marks a region as transparent so the layer below becomes visible. The external view receives that region and renders there.
Before this change, integrating external components such as WebView, Ads, or Camera views had two problems.

1. Gio content could not render above the external view.
2. The external view required an absolute position. That was difficult when combined with offsets or transformations.

The paint.ExternalOp reduces both issues while keeping the API simple. The operation includes a callback that runs every frame. The callback can update position, size, and z-order to keep the native view synchronized with the Gio layout.
Platform behavior required small adjustments, mainly on Windows.

Windows uses an extra child window:
MainWindow → GioWindow

This structure allows additional HWNDs to attach to the MainWindow while GioWindow remains on top. For example:
MainWindow → GioWindow → WebView

ExternalOp exposes a region of the GioWindow so the underlying HWND, such as a WebView, becomes visible in that area.

On macOS, iOS, and Android the rendering stack already uses additional layers. No structural change was required there, only the background was changed to Transparent.

Signed-off-by: inkeliz <inkeliz@inkeliz.com>

Author: inkeliz

app/GioView.java

@@ -16,6 +16,7 @@
 import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.Matrix;
+import android.graphics.PixelFormat;
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
@@ -88,6 +89,11 @@ public GioView(Context context, AttributeSet attrs) {
 		// Set background color to transparent to avoid a flickering
 		// issue on ChromeOS.
 		setBackgroundColor(Color.argb(0, 0, 0, 0));
+		getHolder().setFormat(PixelFormat.TRANSPARENT);
+		
+		// Ensure GioView is on top of other views when ExternalOps are used.
+		// This is initially false and set to true when the first ExternalOp is seen.
+		setZOrderOnTop(false);
 
 		ViewConfiguration conf = ViewConfiguration.get(context);
 		if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
@@ -160,6 +166,22 @@ public GioView(Context context, AttributeSet attrs) {
 			requestUnbufferedDispatch(event);
 		}
 
+		// Check if touch event should be handled by Gio or passed through
+		// to external views. Only check at the beginning of a trace so drags
+		// crossing over external regions are not truncated.
+		if (nhandle != 0) {
+			int action = event.getActionMasked();
+			if (action == MotionEvent.ACTION_DOWN || action == MotionEvent.ACTION_POINTER_DOWN) {
+				int idx = event.getActionIndex();
+				float x = event.getX(idx);
+				float y = event.getY(idx);
+				if (!hitTest(nhandle, x, y)) {
+					// Event is on an external view, don't consume it.
+					return false;
+				}
+			}
+		}
+
 		dispatchMotionEvent(event);
 		return true;
 	}
@@ -578,6 +600,14 @@ void updateCaret(float m00, float m01, float m02, float m10, float m11, float m1
 	static private native int imeToRunes(long handle, int chars);
 	// imeToUTF16 converts the rune index into Java characters.
 	static private native int imeToUTF16(long handle, int runes);
+	// hitTest returns true if the point should be handled by Gio,
+	// false if it should be passed through to external views.
+	static private native boolean hitTest(long handle, float x, float y);
+	// setZOrderOnTop is called when ExternalOps are used to ensure
+	// Gio renders on top of external views.
+	public void setZOrder(boolean onTop) {
+		setZOrderOnTop(onTop);
+	}
 
 	private class GioInputConnection implements InputConnection {
 		private int batchDepth;

app/gl_ios.m

@@ -41,7 +41,9 @@ CFTypeRef gio_createGLLayer(void) {
 		return nil;
 	}
 	layer.drawableProperties = @{kEAGLDrawablePropertyColorFormat: kEAGLColorFormatSRGBA8};
-	layer.opaque = YES;
+	// Enable transparency for external views.
+	layer.opaque = NO;
+	layer.backgroundColor = [UIColor clearColor].CGColor;
 	layer.anchorPoint = CGPointMake(0, 0);
 	return CFBridgingRetain(layer);
 }

app/gl_macos.m

@@ -9,7 +9,11 @@
 
 CALayer *gio_layerFactory(BOOL presentWithTrans) {
 	@autoreleasepool {
-		return [CALayer layer];
+		CALayer *l = [CALayer layer];
+		// Enable transparency for external views.
+		l.opaque = NO;
+		l.backgroundColor = [NSColor clearColor].CGColor;
+		return l;
 	}
 }
 

app/internal/windows/windows.go

@@ -209,6 +209,7 @@ const (
 
 	HWND_TOPMOST = ^(uint32(1) - 1) // -1
 
+	HTTRANSPARENT = ^uintptr(0) // -1, pass through to underlying window
 	HTCAPTION     = 2
 	HTCLIENT      = 1
 	HTLEFT        = 10
@@ -220,6 +221,15 @@ const (
 	HTBOTTOMLEFT  = 16
 	HTBOTTOMRIGHT = 17
 
+	// Region operations for CombineRgn
+	RGN_AND       = 1
+	RGN_OR        = 2
+	RGN_XOR       = 3
+	RGN_DIFF      = 4
+	RGN_COPY      = 5
+	RGN_NULL      = 1
+	RGN_ERROR     = 0
+
 	IDC_APPSTARTING = 32650 // Standard arrow and small hourglass
 	IDC_ARROW       = 32512 // Standard arrow
 	IDC_CROSS       = 32515 // Crosshair
@@ -378,6 +388,7 @@ const (
 	WS_THICKFRAME  = 0x00040000
 	WS_MINIMIZEBOX = 0x00020000
 	WS_MAXIMIZEBOX = 0x00010000
+	WS_CHILD       = 0x40000000
 
 	WS_EX_APPWINDOW  = 0x00040000
 	WS_EX_WINDOWEDGE = 0x00000100
@@ -472,6 +483,7 @@ var (
 	_SetWindowLong32             = user32.NewProc("SetWindowLongW")
 	_SetWindowPlacement          = user32.NewProc("SetWindowPlacement")
 	_SetWindowPos                = user32.NewProc("SetWindowPos")
+	_SetWindowRgn                = user32.NewProc("SetWindowRgn")
 	_SetWindowText               = user32.NewProc("SetWindowTextW")
 	_TranslateMessage            = user32.NewProc("TranslateMessage")
 	_UnregisterClass             = user32.NewProc("UnregisterClassW")
@@ -480,8 +492,11 @@ var (
 	shcore            = syscall.NewLazySystemDLL("shcore")
 	_GetDpiForMonitor = shcore.NewProc("GetDpiForMonitor")
 
-	gdi32          = syscall.NewLazySystemDLL("gdi32")
-	_GetDeviceCaps = gdi32.NewProc("GetDeviceCaps")
+	gdi32               = syscall.NewLazySystemDLL("gdi32")
+	_GetDeviceCaps      = gdi32.NewProc("GetDeviceCaps")
+	_CreateRectRgn      = gdi32.NewProc("CreateRectRgn")
+	_CombineRgn         = gdi32.NewProc("CombineRgn")
+	_DeleteObject       = gdi32.NewProc("DeleteObject")
 
 	imm32                    = syscall.NewLazySystemDLL("imm32")
 	_ImmGetContext           = imm32.NewProc("ImmGetContext")
@@ -993,3 +1008,32 @@ func (p *WindowPlacement) Set(Left, Top, Right, Bottom int) {
 	p.rcNormalPosition.Right = int32(Right)
 	p.rcNormalPosition.Bottom = int32(Bottom)
 }
+
+// CreateRectRgn creates a rectangular region.
+func CreateRectRgn(left, top, right, bottom int32) syscall.Handle {
+	r, _, _ := _CreateRectRgn.Call(uintptr(left), uintptr(top), uintptr(right), uintptr(bottom))
+	return syscall.Handle(r)
+}
+
+// CombineRgn combines two regions.
+func CombineRgn(dst, src1, src2 syscall.Handle, mode int) int {
+	r, _, _ := _CombineRgn.Call(uintptr(dst), uintptr(src1), uintptr(src2), uintptr(mode))
+	return int(r)
+}
+
+// DeleteObject deletes a GDI object.
+func DeleteObject(h syscall.Handle) bool {
+	r, _, _ := _DeleteObject.Call(uintptr(h))
+	return r != 0
+}
+
+// SetWindowRgn sets the window region for hit testing.
+// Pass 0 as hRgn to reset to the full window rect.
+func SetWindowRgn(hwnd syscall.Handle, hRgn syscall.Handle, redraw bool) bool {
+	var redrawInt uintptr
+	if redraw {
+		redrawInt = 1
+	}
+	r, _, _ := _SetWindowRgn.Call(uintptr(hwnd), uintptr(hRgn), redrawInt)
+	return r != 0
+}

app/metal_ios.go

@@ -24,6 +24,9 @@ static CFTypeRef getMetalLayer(CFTypeRef viewRef) {
 		CAMetalLayer *l = (CAMetalLayer *)view.layer;
 		l.needsDisplayOnBoundsChange = YES;
 		l.presentsWithTransaction = YES;
+		// Enable transparency for external views.
+		l.opaque = NO;
+		l.backgroundColor = [UIColor clearColor].CGColor;
 		return CFBridgingRetain(l);
 	}
 }

app/metal_macos.go

@@ -18,6 +18,9 @@ CALayer *gio_layerFactory(BOOL presentWithTrans) {
 		l.autoresizingMask = kCALayerHeightSizable|kCALayerWidthSizable;
 		l.needsDisplayOnBoundsChange = YES;
 		l.presentsWithTransaction = presentWithTrans;
+		// Enable transparency for external views.
+		l.opaque = NO;
+		l.backgroundColor = [NSColor clearColor].CGColor;
 		return l;
 	}
 }

app/os.go

@@ -205,6 +205,8 @@ type driver interface {
 	Frame(frame *op.Ops)
 	// ProcessEvent processes an event.
 	ProcessEvent(e event.Event)
+	// SetExternalRegions updates regions for external view hit testing.
+	SetExternalRegions(gpu.ExternalRegions)
 }
 
 type windowRendezvous struct {

app/os_android.go

@@ -137,6 +137,7 @@ import (
 	"unicode/utf16"
 	"unsafe"
 
+	"gioui.org/gpu"
 	"gioui.org/internal/f32color"
 	"gioui.org/op"
 
@@ -176,6 +177,11 @@ type window struct {
 		focusID input.SemanticID
 		diffs   []input.SemanticID
 	}
+
+	// externalRegions tracks areas for hit testing.
+	externalRegions gpu.ExternalRegions
+	// externalUsed tracks whether ExternalOps have been used this frame.
+	externalUsed bool
 }
 
 // gioView hold cached JNI methods for GioView.
@@ -200,6 +206,7 @@ var gioView struct {
 	restartInput       C.jmethodID
 	updateSelection    C.jmethodID
 	updateCaret        C.jmethodID
+	setZOrder          C.jmethodID
 }
 
 type pixelInsets struct {
@@ -486,6 +493,7 @@ func Java_org_gioui_GioView_onCreateView(env *C.JNIEnv, class C.jclass, view C.j
 		m.restartInput = getMethodID(env, class, "restartInput", "()V")
 		m.updateSelection = getMethodID(env, class, "updateSelection", "()V")
 		m.updateCaret = getMethodID(env, class, "updateCaret", "(FFFFFFFFFF)V")
+		m.setZOrder = getMethodID(env, class, "setZOrder", "(Z)V")
 	})
 	view = C.jni_NewGlobalRef(env, view)
 	wopts := <-mainWindow.out
@@ -992,6 +1000,27 @@ func Java_org_gioui_GioView_onKeyEvent(env *C.JNIEnv, class C.jclass, handle C.j
 	}
 }
 
+// SetExternalRegions updates the external regions for hit testing.
+func (w *window) SetExternalRegions(regions gpu.ExternalRegions) {
+	w.externalRegions = regions
+	// If ExternalOps are used and z-order hasn't been set yet, set GioView on top.
+	if len(regions.Pass) > 0 && !w.externalUsed {
+		w.externalUsed = true
+		runInJVM(javaVM(), func(env *C.JNIEnv) {
+			callVoidMethod(env, w.view, gioView.setZOrder, jvalue(1))
+		})
+	}
+}
+
+//export Java_org_gioui_GioView_hitTest
+func Java_org_gioui_GioView_hitTest(env *C.JNIEnv, class C.jclass, handle C.jlong, x, y C.jfloat) C.jboolean {
+	w := cgo.Handle(handle).Value().(*window)
+	if w.externalRegions.Contains(f32.Point{X: float32(x), Y: float32(y)}) {
+		return C.JNI_FALSE
+	}
+	return C.JNI_TRUE
+}
+
 //export Java_org_gioui_GioView_onTouchEvent
 func Java_org_gioui_GioView_onTouchEvent(env *C.JNIEnv, class C.jclass, handle C.jlong, action, pointerID, tool C.jint, x, y, scrollX, scrollY C.jfloat, jbtns C.jint, t C.jlong) {
 	w := cgo.Handle(handle).Value().(*window)

app/os_ios.go

@@ -14,6 +14,7 @@ package app
 
 __attribute__ ((visibility ("hidden"))) int gio_applicationMain(int argc, char *argv[]);
 __attribute__ ((visibility ("hidden"))) void gio_viewSetHandle(CFTypeRef viewRef, uintptr_t handle);
+__attribute__ ((visibility ("hidden"))) void gio_setZOrderOnTop(CFTypeRef viewRef, int onTop);
 
 struct drawParams {
 	CGFloat dpi, sdpi;
@@ -92,6 +93,7 @@ import (
 	"unsafe"
 
 	"gioui.org/f32"
+	"gioui.org/gpu"
 	"gioui.org/io/event"
 	"gioui.org/io/key"
 	"gioui.org/io/pointer"
@@ -117,6 +119,19 @@ type window struct {
 	config Config
 
 	pointerMap []C.CFTypeRef
+
+	externalRegions gpu.ExternalRegions
+	externalUsed bool
+}
+
+// SetExternalRegions updates the external regions for hit testing.
+func (w *window) SetExternalRegions(regions gpu.ExternalRegions) {
+	w.externalRegions = regions
+	// If ExternalOps are used and z-order hasn't been set yet, bring GioView to front.
+	if len(regions.Pass) > 0 && !w.externalUsed {
+		w.externalUsed = true
+		C.gio_setZOrderOnTop(w.view, 1)
+	}
 }
 
 var mainWindow = newWindowRendezvous()
@@ -153,6 +168,15 @@ func viewFor(h C.uintptr_t) *window {
 	return cgo.Handle(h).Value().(*window)
 }
 
+//export gio_hitTest
+func gio_hitTest(h C.uintptr_t, x, y C.CGFloat) C.int {
+	w := viewFor(h)
+	if w.externalRegions.Contains(f32.Point{X: float32(x), Y: float32(y)}) {
+		return 0
+	}
+	return 1
+}
+
 //export gio_onDraw
 func gio_onDraw(h C.uintptr_t) {
 	w := viewFor(h)

app/os_ios.m

@@ -9,6 +9,8 @@
 #include "framework_ios.h"
 
 __attribute__ ((visibility ("hidden"))) Class gio_layerClass(void);
+__attribute__ ((visibility ("hidden"))) int gio_hitTest(uintptr_t handle, CGFloat x, CGFloat y);
+__attribute__ ((visibility ("hidden"))) void gio_setZOrderOnTop(CFTypeRef viewRef, int onTop);
 
 @interface GioView: UIView <UIKeyInput>
 @property uintptr_t handle;
@@ -133,6 +135,16 @@ + (void)onFrameCallback:(CADisplayLink *)link {
 + (Class)layerClass {
     return gio_layerClass();
 }
+- (instancetype)initWithFrame:(CGRect)frame {
+    self = [super initWithFrame:frame];
+    if (self) {
+        // Ensure user interaction is enabled for hit testing.
+        self.userInteractionEnabled = YES;
+        // Ensure the view is opaque to touch events.
+        self.opaque = NO;
+    }
+    return self;
+}
 - (void)willMoveToWindow:(UIWindow *)newWindow {
 	if (self.window != nil) {
 		[[NSNotificationCenter defaultCenter] removeObserver:self
@@ -181,6 +193,39 @@ - (void)touchesCancelled:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event
 	handleTouches(1, self, touches, event);
 }
 
+// hitTest returns true if the point should be handled by Gio,
+// false if it should be passed through to external views.
+- (BOOL)hitTestPoint:(CGPoint)point {
+	if (self.handle == 0) {
+		return YES;
+	}
+	return gio_hitTest(self.handle, point.x * self.contentScaleFactor, point.y * self.contentScaleFactor);
+}
+
+- (UIView *)hitTest:(CGPoint)point withEvent:(UIEvent *)event {
+	// Check if this point is within an external view's bounds.
+	// If so, return nil to allow the event to pass through.
+	if (![self hitTestPoint:point]) {
+		return nil;
+	}
+	return [super hitTest:point withEvent:event];
+}
+
+- (BOOL)pointInside:(CGPoint)point withEvent:(UIEvent *)event {
+	// Allow touch events to pass through if the point is in an external region.
+	if (![self hitTestPoint:point]) {
+		return NO;
+	}
+	return [super pointInside:point withEvent:event];
+}
+
+// setZOrderOnTop brings this view to the front of its superview.
+- (void)setZOrderOnTop:(BOOL)onTop {
+	if (onTop && self.superview != nil) {
+		[self.superview bringSubviewToFront:self];
+	}
+}
+
 - (void)insertText:(NSString *)text {
 	onText(self.handle, (__bridge CFTypeRef)text);
 }
@@ -281,6 +326,11 @@ void gio_viewSetHandle(CFTypeRef viewRef, uintptr_t handle) {
 	v.handle = handle;
 }
 
+void gio_setZOrderOnTop(CFTypeRef viewRef, int onTop) {
+	GioView *v = (__bridge GioView *)viewRef;
+	[v setZOrderOnTop:(onTop != 0)];
+}
+
 @interface _gioAppDelegate : UIResponder <UIApplicationDelegate>
 @property (strong, nonatomic) UIWindow *window;
 @end