feat: high resolution screenshot from viewer (#646)

Co-authored-by: vidhya-metacell <vidhya@metacell.us>
Co-authored-by: Aiga115 <aiguljhyldyzbekkyzy@gmail.com>

Author: 3 people

python/neuroglancer/tool/screenshot.py

@@ -596,6 +596,11 @@ def define_state_modification_args(ap: argparse.ArgumentParser):
         type=float,
         help="Multiply projection view scale by specified factor.",
     )
+    ap.add_argument(
+        "--resolution-scale-factor",
+        type=float,
+        help="Divide cross section view scale by specified factor. E.g. a 2000x2000 output with a resolution scale factor of 2 will have the same FOV as a 1000x1000 output.",
+    )
     ap.add_argument(
         "--system-memory-limit",
         type=int,
@@ -635,6 +640,8 @@ def apply_state_modifications(
         state.show_default_annotations = args.show_default_annotations
     if args.projection_scale_multiplier is not None:
         state.projection_scale *= args.projection_scale_multiplier
+    if args.resolution_scale_factor is not None:
+        state.cross_section_scale /= args.resolution_scale_factor
     if args.cross_section_background_color is not None:
         state.cross_section_background_color = args.cross_section_background_color
 

python/neuroglancer/viewer_config_state.py

@@ -97,6 +97,34 @@ class LayerSelectedValues(_LayerSelectedValuesBase):
     """Specifies the data values associated with the current mouse position."""
 
 
+@export
+class PanelResolutionData(JsonObjectWrapper):
+    __slots__ = ()
+    type = wrapped_property("type", str)
+    width = wrapped_property("width", int)
+    height = wrapped_property("height", int)
+    resolution = wrapped_property("resolution", str)
+
+
+@export
+class LayerResolutionData(JsonObjectWrapper):
+    __slots__ = ()
+    name = wrapped_property("name", str)
+    type = wrapped_property("type", str)
+    resolution = wrapped_property("resolution", str)
+
+
+@export
+class ScreenshotResolutionMetadata(JsonObjectWrapper):
+    __slots__ = ()
+    panel_resolution_data = panelResolutionData = wrapped_property(
+        "panelResolutionData", typed_list(PanelResolutionData)
+    )
+    layer_resolution_data = layerResolutionData = wrapped_property(
+        "layerResolutionData", typed_list(LayerResolutionData)
+    )
+
+
 @export
 class ScreenshotReply(JsonObjectWrapper):
     __slots__ = ()
@@ -106,6 +134,9 @@ class ScreenshotReply(JsonObjectWrapper):
     height = wrapped_property("height", int)
     image_type = imageType = wrapped_property("imageType", str)
     depth_data = depthData = wrapped_property("depthData", optional(base64.b64decode))
+    resolution_metadata = resolutionMetadata = wrapped_property(
+        "resolutionMetadata", ScreenshotResolutionMetadata
+    )
 
     @property
     def image_pixels(self):

src/display_context.ts

@@ -25,6 +25,10 @@ import { FramerateMonitor } from "#src/util/framerate.js";
 import type { mat4 } from "#src/util/geom.js";
 import { parseFixedLengthArray, verifyFloat01 } from "#src/util/json.js";
 import { NullarySignal } from "#src/util/signal.js";
+import {
+  TrackableScreenshotMode,
+  ScreenshotMode,
+} from "#src/util/trackable_screenshot_mode.js";
 import type { WatchableVisibilityPriority } from "#src/visibility_priority/frontend.js";
 import type { GL } from "#src/webgl/context.js";
 import { initializeWebGL } from "#src/webgl/context.js";
@@ -135,7 +139,7 @@ export abstract class RenderedPanel extends RefCounted {
 
   abstract isReady(): boolean;
 
-  ensureBoundsUpdated() {
+  ensureBoundsUpdated(canScaleForScreenshot: boolean = false) {
     const { context } = this;
     context.ensureBoundsUpdated();
     const { boundsGeneration } = context;
@@ -221,8 +225,18 @@ export abstract class RenderedPanel extends RefCounted {
       0,
       clippedBottom - clippedTop,
     ));
-    viewport.logicalWidth = logicalWidth;
-    viewport.logicalHeight = logicalHeight;
+    if (
+      this.context.screenshotMode.value !== ScreenshotMode.OFF &&
+      canScaleForScreenshot
+    ) {
+      viewport.width = logicalWidth * screenToCanvasPixelScaleX;
+      viewport.height = logicalHeight * screenToCanvasPixelScaleY;
+      viewport.logicalWidth = logicalWidth * screenToCanvasPixelScaleX;
+      viewport.logicalHeight = logicalHeight * screenToCanvasPixelScaleY;
+    } else {
+      viewport.logicalWidth = logicalWidth;
+      viewport.logicalHeight = logicalHeight;
+    }
     viewport.visibleLeftFraction = (clippedLeft - logicalLeft) / logicalWidth;
     viewport.visibleTopFraction = (clippedTop - logicalTop) / logicalHeight;
     viewport.visibleWidthFraction = clippedWidth / logicalWidth;
@@ -410,6 +424,9 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
   rootRect: DOMRect | undefined;
   resizeGeneration = 0;
   boundsGeneration = -1;
+  screenshotMode: TrackableScreenshotMode = new TrackableScreenshotMode(
+    ScreenshotMode.OFF,
+  );
   force3DHistogramForAutoRange = false;
   private framerateMonitor = new FramerateMonitor();
 
@@ -599,8 +616,10 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
     const { resizeGeneration } = this;
     if (this.boundsGeneration === resizeGeneration) return;
     const { canvas } = this;
-    canvas.width = canvas.offsetWidth;
-    canvas.height = canvas.offsetHeight;
+    if (this.screenshotMode.value === ScreenshotMode.OFF) {
+      canvas.width = canvas.offsetWidth;
+      canvas.height = canvas.offsetHeight;
+    }
     this.canvasRect = canvas.getBoundingClientRect();
     this.rootRect = this.container.getBoundingClientRect();
     this.boundsGeneration = resizeGeneration;

src/overlay.ts

@@ -47,11 +47,15 @@ export class Overlay extends RefCounted {
     document.body.appendChild(container);
     this.registerDisposer(new KeyboardEventBinder(this.container, this.keyMap));
     this.registerEventListener(container, "action:close", () => {
-      this.dispose();
+      this.close();
     });
     content.focus();
   }
 
+  close() {
+    this.dispose();
+  }
+
   disposed() {
     --overlaysOpen;
     document.body.removeChild(this.container);

src/perspective_view/panel.ts

@@ -605,7 +605,7 @@ export class PerspectivePanel extends RenderedDataPanel {
   }
 
   ensureBoundsUpdated() {
-    super.ensureBoundsUpdated();
+    super.ensureBoundsUpdated(true /* canScaleForScreenshot */);
     this.projectionParameters.setViewport(this.renderViewport);
   }
 

src/python_integration/screenshots.ts

@@ -28,11 +28,54 @@ import { convertEndian32, Endianness } from "#src/util/endian.js";
 import { verifyOptionalString } from "#src/util/json.js";
 import { Signal } from "#src/util/signal.js";
 import { getCachedJson } from "#src/util/trackable.js";
+import { ScreenshotMode } from "#src/util/trackable_screenshot_mode.js";
+import type { ResolutionMetadata } from "#src/util/viewer_resolution_stats.js";
+import { getViewerResolutionMetadata } from "#src/util/viewer_resolution_stats.js";
 import type { Viewer } from "#src/viewer.js";
 
+export interface ScreenshotResult {
+  id: string;
+  image: string;
+  imageType: string;
+  depthData: string | undefined;
+  width: number;
+  height: number;
+  resolutionMetadata: ResolutionMetadata;
+}
+
+export interface ScreenshotActionState {
+  viewerState: any;
+  selectedValues: any;
+  screenshot: ScreenshotResult;
+}
+
+export interface ScreenshotChunkStatistics {
+  downloadLatency: number;
+  visibleChunksDownloading: number;
+  visibleChunksFailed: number;
+  visibleChunksGpuMemory: number;
+  visibleChunksSystemMemory: number;
+  visibleChunksTotal: number;
+  visibleGpuMemory: number;
+}
+
+export interface StatisticsActionState {
+  viewerState: any;
+  selectedValues: any;
+  screenshotStatistics: {
+    id: string;
+    chunkSources: any[];
+    total: ScreenshotChunkStatistics;
+  };
+}
+
 export class ScreenshotHandler extends RefCounted {
-  sendScreenshotRequested = new Signal<(state: any) => void>();
-  sendStatisticsRequested = new Signal<(state: any) => void>();
+  sendScreenshotRequested = new Signal<
+    (state: ScreenshotActionState) => void
+  >();
+  sendStatisticsRequested = new Signal<
+    (state: StatisticsActionState) => void
+  >();
   requestState = new TrackableValue<string | undefined>(
     undefined,
     verifyOptionalString,
@@ -124,12 +167,14 @@ export class ScreenshotHandler extends RefCounted {
       return;
     }
     const { viewer } = this;
-    if (!viewer.isReady()) {
+    const shouldForceScreenshot =
+      this.viewer.display.screenshotMode.value === ScreenshotMode.FORCE;
+    if (!viewer.isReady() && !shouldForceScreenshot) {
       this.wasAlreadyVisible = false;
       this.throttledSendStatistics(requestState);
       return;
     }
-    if (!this.wasAlreadyVisible) {
+    if (!this.wasAlreadyVisible && !shouldForceScreenshot) {
       this.throttledSendStatistics(requestState);
       this.wasAlreadyVisible = true;
       this.debouncedMaybeSendScreenshot();
@@ -140,6 +185,7 @@ export class ScreenshotHandler extends RefCounted {
     this.throttledSendStatistics.cancel();
     viewer.display.draw();
     const screenshotData = viewer.display.canvas.toDataURL();
+    const resolutionMetadata = getViewerResolutionMetadata(viewer);
     const { width, height } = viewer.display.canvas;
     const prefix = "data:image/png;base64,";
     let imageType: string;
@@ -169,6 +215,7 @@ export class ScreenshotHandler extends RefCounted {
         depthData,
         width,
         height,
+        resolutionMetadata,
       },
     };
 

src/single_mesh/frontend.ts

@@ -20,8 +20,18 @@ import {
   ChunkSource,
   WithParameters,
 } from "#src/chunk_manager/frontend.js";
+import {
+  makeCoordinateSpace,
+  makeIdentityTransform,
+} from "#src/coordinate_transform.js";
+import type {
+  DataSource,
+  GetKvStoreBasedDataSourceOptions,
+  KvStoreBasedDataSourceProvider,
+} from "#src/datasource/index.js";
 import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js";
 import type { SharedKvStoreContext } from "#src/kvstore/frontend.js";
+import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js";
 import type { PickState, VisibleLayerInfo } from "#src/layer/index.js";
 import type { PerspectivePanel } from "#src/perspective_view/panel.js";
 import type { PerspectiveViewRenderContext } from "#src/perspective_view/render_layer.js";
@@ -85,16 +95,6 @@ import {
   TextureFormat,
 } from "#src/webgl/texture_access.js";
 import { SharedObject } from "#src/worker_rpc.js";
-import type {
-  DataSource,
-  GetKvStoreBasedDataSourceOptions,
-  KvStoreBasedDataSourceProvider,
-} from "#src/datasource/index.js";
-import { ensureEmptyUrlSuffix } from "#src/kvstore/url.js";
-import {
-  makeCoordinateSpace,
-  makeIdentityTransform,
-} from "#src/coordinate_transform.js";
 
 const DEFAULT_FRAGMENT_MAIN = `void main() {
   emitGray();

src/sliceview/panel.ts

@@ -435,7 +435,7 @@ export class SliceViewPanel extends RenderedDataPanel {
   }
 
   ensureBoundsUpdated() {
-    super.ensureBoundsUpdated();
+    super.ensureBoundsUpdated(true /* canScaleForScreenshot */);
     this.sliceView.projectionParameters.setViewport(this.renderViewport);
   }
 

src/sliceview/volume/renderlayer.ts

@@ -341,6 +341,7 @@ export abstract class SliceViewVolumeRenderLayer<
   >;
   private tempChunkPosition: Float32Array;
   shaderParameters: WatchableValueInterface<ShaderParameters>;
+  highestResolutionLoadedVoxelSize: Float32Array | undefined;
   private vertexIdHelper: VertexIdHelper;
 
   constructor(
@@ -570,6 +571,7 @@ void main() {
         this.chunkManager.chunkQueueManager.frameNumberCounter.frameNumber,
       );
     }
+    this.highestResolutionLoadedVoxelSize = undefined;
 
     let shaderResult: ParameterizedShaderGetterResult<
       ShaderParameters,
@@ -692,6 +694,18 @@ void main() {
           effectiveVoxelSize[1],
           effectiveVoxelSize[2],
         );
+        if (presentCount > 0) {
+          const medianStoredVoxelSize = this.highestResolutionLoadedVoxelSize
+            ? medianOf3(
+                this.highestResolutionLoadedVoxelSize[0],
+                this.highestResolutionLoadedVoxelSize[1],
+                this.highestResolutionLoadedVoxelSize[2],
+              )
+            : Infinity;
+          if (medianVoxelSize <= medianStoredVoxelSize) {
+            this.highestResolutionLoadedVoxelSize = effectiveVoxelSize;
+          }
+        }
         renderScaleHistogram.add(
           medianVoxelSize,
           medianVoxelSize / projectionParameters.pixelSize,