diff --git a/Android.bp b/Android.bp index 7493b53899..82e6e2a4dd 100644 --- a/Android.bp +++ b/Android.bp @@ -3467,6 +3467,7 @@ filegroup { "protos/perfetto/trace/android/surfaceflinger_transactions.proto", "protos/perfetto/trace/android/typedef.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", "protos/perfetto/trace/android/view/display.proto", "protos/perfetto/trace/android/view/displaycutout.proto", "protos/perfetto/trace/android/view/displayinfo.proto", @@ -7626,6 +7627,7 @@ filegroup { "protos/perfetto/trace/android/packages_list.proto", "protos/perfetto/trace/android/pixel_modem_events.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", ], } @@ -7662,6 +7664,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.gen.cc", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.gen.cc", "external/perfetto/protos/perfetto/trace/android/user_list.gen.cc", + "external/perfetto/protos/perfetto/trace/android/video_frame.gen.cc", ], } @@ -7698,6 +7701,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.gen.h", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.gen.h", "external/perfetto/protos/perfetto/trace/android/user_list.gen.h", + "external/perfetto/protos/perfetto/trace/android/video_frame.gen.h", ], export_include_dirs: [ ".", @@ -7726,6 +7730,7 @@ filegroup { "protos/perfetto/trace/android/packages_list.proto", "protos/perfetto/trace/android/pixel_modem_events.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", ], } @@ -7761,6 +7766,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.pb.cc", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.pb.cc", "external/perfetto/protos/perfetto/trace/android/user_list.pb.cc", + "external/perfetto/protos/perfetto/trace/android/video_frame.pb.cc", ], } @@ -7796,6 +7802,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.pb.h", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.pb.h", "external/perfetto/protos/perfetto/trace/android/user_list.pb.h", + "external/perfetto/protos/perfetto/trace/android/video_frame.pb.h", ], export_include_dirs: [ ".", @@ -8415,6 +8422,7 @@ filegroup { "protos/perfetto/trace/android/packages_list.proto", "protos/perfetto/trace/android/pixel_modem_events.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", ], } @@ -8451,6 +8459,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.pbzero.cc", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.pbzero.cc", "external/perfetto/protos/perfetto/trace/android/user_list.pbzero.cc", + "external/perfetto/protos/perfetto/trace/android/video_frame.pbzero.cc", ], } @@ -8487,6 +8496,7 @@ genrule { "external/perfetto/protos/perfetto/trace/android/packages_list.pbzero.h", "external/perfetto/protos/perfetto/trace/android/pixel_modem_events.pbzero.h", "external/perfetto/protos/perfetto/trace/android/user_list.pbzero.h", + "external/perfetto/protos/perfetto/trace/android/video_frame.pbzero.h", ], export_include_dirs: [ ".", @@ -8760,6 +8770,7 @@ genrule { "protos/perfetto/trace/android/surfaceflinger_layers.proto", "protos/perfetto/trace/android/surfaceflinger_transactions.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", "protos/perfetto/trace/android/winscope_extensions.proto", "protos/perfetto/trace/chrome/chrome_benchmark_metadata.proto", "protos/perfetto/trace/chrome/chrome_metadata.proto", @@ -15904,6 +15915,7 @@ filegroup { "src/trace_processor/importers/proto/track_event_tokenizer.cc", "src/trace_processor/importers/proto/track_event_tracker.cc", "src/trace_processor/importers/proto/user_tracker.cc", + "src/trace_processor/importers/proto/video_frame_module.cc", ], } @@ -16394,6 +16406,7 @@ filegroup { "src/trace_processor/perfetto_sql/intrinsics/functions/structural_tree_partition.cc", "src/trace_processor/perfetto_sql/intrinsics/functions/to_ftrace.cc", "src/trace_processor/perfetto_sql/intrinsics/functions/type_builders.cc", + "src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.cc", ], } @@ -16689,6 +16702,7 @@ genrule { "src/trace_processor/perfetto_sql/stdlib/android/thread.sql", "src/trace_processor/perfetto_sql/stdlib/android/user_list.sql", "src/trace_processor/perfetto_sql/stdlib/android/version.sql", + "src/trace_processor/perfetto_sql/stdlib/android/video_frames.sql", "src/trace_processor/perfetto_sql/stdlib/android/wakeups.sql", "src/trace_processor/perfetto_sql/stdlib/android/winscope/inputmethod.sql", "src/trace_processor/perfetto_sql/stdlib/android/winscope/rect.sql", @@ -19003,6 +19017,7 @@ java_library { "protos/perfetto/trace/android/surfaceflinger_layers.proto", "protos/perfetto/trace/android/surfaceflinger_transactions.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", "protos/perfetto/trace/android/winscope_extensions.proto", "protos/perfetto/trace/chrome/chrome_benchmark_metadata.proto", "protos/perfetto/trace/chrome/chrome_metadata.proto", diff --git a/BUILD b/BUILD index aa15dee6bc..11b0929b64 100644 --- a/BUILD +++ b/BUILD @@ -2952,6 +2952,8 @@ perfetto_filegroup( "src/trace_processor/importers/proto/track_event_tracker.h", "src/trace_processor/importers/proto/user_tracker.cc", "src/trace_processor/importers/proto/user_tracker.h", + "src/trace_processor/importers/proto/video_frame_module.cc", + "src/trace_processor/importers/proto/video_frame_module.h", ], ) @@ -3392,6 +3394,8 @@ perfetto_filegroup( "src/trace_processor/perfetto_sql/intrinsics/functions/type_builders.cc", "src/trace_processor/perfetto_sql/intrinsics/functions/type_builders.h", "src/trace_processor/perfetto_sql/intrinsics/functions/utils.h", + "src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.cc", + "src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h", "src/trace_processor/perfetto_sql/intrinsics/functions/window_functions.h", ], ) @@ -3718,6 +3722,7 @@ perfetto_filegroup( "src/trace_processor/perfetto_sql/stdlib/android/thread.sql", "src/trace_processor/perfetto_sql/stdlib/android/user_list.sql", "src/trace_processor/perfetto_sql/stdlib/android/version.sql", + "src/trace_processor/perfetto_sql/stdlib/android/video_frames.sql", "src/trace_processor/perfetto_sql/stdlib/android/wakeups.sql", ], ) @@ -7265,6 +7270,7 @@ perfetto_proto_library( "protos/perfetto/trace/android/packages_list.proto", "protos/perfetto/trace/android/pixel_modem_events.proto", "protos/perfetto/trace/android/user_list.proto", + "protos/perfetto/trace/android/video_frame.proto", ], visibility = [ PERFETTO_CONFIG.proto_library_visibility, diff --git a/protos/perfetto/trace/android/BUILD.gn b/protos/perfetto/trace/android/BUILD.gn index 7deb92232f..4b9fe55b0b 100644 --- a/protos/perfetto/trace/android/BUILD.gn +++ b/protos/perfetto/trace/android/BUILD.gn @@ -38,6 +38,7 @@ perfetto_proto_library("@TYPE@") { "packages_list.proto", "pixel_modem_events.proto", "user_list.proto", + "video_frame.proto", ] } diff --git a/protos/perfetto/trace/android/video_frame.proto b/protos/perfetto/trace/android/video_frame.proto new file mode 100644 index 0000000000..f1a3b05a37 --- /dev/null +++ b/protos/perfetto/trace/android/video_frame.proto @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package perfetto.protos; + +// A captured video frame (screenshot) from the device display. +// Used by the android.video_frames data source for continuous frame capture. +message VideoFrame { + // Sequential frame number within the capture session. + optional uint64 frame_number = 1; + + // JPEG-compressed image data. + optional bytes jpg_image = 2; + + // Display name for the track in the UI (e.g. "Front Camera"). + optional string track_name = 3; + + // Unique identifier for the stream. Frames with the same track_id are + // grouped into the same track. + optional uint32 track_id = 4; + + // WebP-compressed image data. Used when jpg_image is not set. + optional bytes webp_image = 5; +} diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto index 46a212c390..da6792ae9b 100644 --- a/protos/perfetto/trace/perfetto_trace.proto +++ b/protos/perfetto/trace/perfetto_trace.proto @@ -7599,6 +7599,30 @@ message AndroidUserList { // End of protos/perfetto/trace/android/user_list.proto +// Begin of protos/perfetto/trace/android/video_frame.proto + +// A captured video frame (screenshot) from the device display. +// Used by the android.video_frames data source for continuous frame capture. +message VideoFrame { + // Sequential frame number within the capture session. + optional uint64 frame_number = 1; + + // JPEG-compressed image data. + optional bytes jpg_image = 2; + + // Display name for the track in the UI (e.g. "Front Camera"). + optional string track_name = 3; + + // Unique identifier for the stream. Frames with the same track_id are + // grouped into the same track. + optional uint32 track_id = 4; + + // WebP-compressed image data. Used when jpg_image is not set. + optional bytes webp_image = 5; +} + +// End of protos/perfetto/trace/android/video_frame.proto + // Begin of protos/perfetto/trace/android/winscope_extensions.proto message WinscopeExtensions { @@ -18723,6 +18747,8 @@ message TracePacket { AndroidUserList user_list = 123; + VideoFrame video_frame = 129; + // This field is only used for testing. // In previous versions of this proto this field had the id 268435455 // This caused many problems: diff --git a/protos/perfetto/trace/trace_packet.proto b/protos/perfetto/trace/trace_packet.proto index b8c5f0c159..0d07f3b57a 100644 --- a/protos/perfetto/trace/trace_packet.proto +++ b/protos/perfetto/trace/trace_packet.proto @@ -43,6 +43,7 @@ import "protos/perfetto/trace/android/shell_transition.proto"; import "protos/perfetto/trace/android/surfaceflinger_layers.proto"; import "protos/perfetto/trace/android/surfaceflinger_transactions.proto"; import "protos/perfetto/trace/android/user_list.proto"; +import "protos/perfetto/trace/android/video_frame.proto"; import "protos/perfetto/trace/android/winscope_extensions.proto"; import "protos/perfetto/trace/chrome/chrome_benchmark_metadata.proto"; import "protos/perfetto/trace/chrome/chrome_metadata.proto"; @@ -296,6 +297,8 @@ message TracePacket { AndroidUserList user_list = 123; + VideoFrame video_frame = 129; + // This field is only used for testing. // In previous versions of this proto this field had the id 268435455 // This caused many problems: diff --git a/src/java_datasource/android_video_frames/README.md b/src/java_datasource/android_video_frames/README.md new file mode 100644 index 0000000000..25f6a87ddd --- /dev/null +++ b/src/java_datasource/android_video_frames/README.md @@ -0,0 +1,37 @@ +# Android Video Frames Data Source + +## Overview + +A Perfetto data source that captures screen frames as JPEG images and stores +them as `VideoFrame` trace packets with native BLOB storage. + +This data source lives in the Android tree (`frameworks/base`), not in this +repo. The Perfetto-side changes (trace processor module, UI plugin, tests) are +in this repo. + +## Test data + +- Trace generator: `test/trace_processor/diff_tests/parser/android/generate_video_trace.py` +- Diff tests: `test/trace_processor/diff_tests/parser/android/tests.py` + (see `test_video_frame_*`) + +## Proto + +`VideoFrame` is field 129 on `TracePacket`, defined in +`protos/perfetto/trace/android/video_frame.proto`. + +Fields: +- `frame_number` (uint64): sequential frame number +- `jpg_image` (bytes): JPEG image data +- `track_name` (string): display name for the UI track +- `track_id` (uint32): groups frames into streams + +## Trace config example + +```proto +data_sources { + config { + name: "android.video_frames" + } +} +``` diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index 0a2117e3c7..eb652f74df 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn @@ -72,6 +72,8 @@ source_set("minimal") { "track_event_tracker.h", "user_tracker.cc", "user_tracker.h", + "video_frame_module.cc", + "video_frame_module.h", ] public_deps = [ ":proto_importer_module" ] deps = [ diff --git a/src/trace_processor/importers/proto/additional_modules.cc b/src/trace_processor/importers/proto/additional_modules.cc index 3243472d04..0ad56084cb 100644 --- a/src/trace_processor/importers/proto/additional_modules.cc +++ b/src/trace_processor/importers/proto/additional_modules.cc @@ -42,6 +42,7 @@ #include "src/trace_processor/importers/proto/trace.descriptor.h" #include "src/trace_processor/importers/proto/translation_table_module.h" #include "src/trace_processor/importers/proto/v8_module.h" +#include "src/trace_processor/importers/proto/video_frame_module.h" #include "src/trace_processor/types/trace_processor_context.h" #if PERFETTO_BUILDFLAG(PERFETTO_ENABLE_WINSCOPE) @@ -89,6 +90,8 @@ void RegisterAdditionalModules(ProtoImporterModuleContext* module_context, new AppWakelockModule(module_context, context)); module_context->modules.emplace_back( new GenericKernelModule(module_context, context)); + module_context->modules.emplace_back( + new VideoFrameModule(module_context, context)); #if PERFETTO_BUILDFLAG(PERFETTO_ENABLE_WINSCOPE) module_context->modules.emplace_back( diff --git a/src/trace_processor/importers/proto/video_frame_module.cc b/src/trace_processor/importers/proto/video_frame_module.cc new file mode 100644 index 0000000000..c54a739217 --- /dev/null +++ b/src/trace_processor/importers/proto/video_frame_module.cc @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/video_frame_module.h" + +#include + +#include "perfetto/trace_processor/trace_blob_view.h" +#include "protos/perfetto/trace/android/video_frame.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/trace_processor/importers/common/parser_types.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/android_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor { + +using protos::pbzero::TracePacket; + +VideoFrameModule::~VideoFrameModule() = default; + +VideoFrameModule::VideoFrameModule(ProtoImporterModuleContext* module_context, + TraceProcessorContext* context) + : ProtoImporterModule(module_context), context_(context) { + RegisterForField(TracePacket::kVideoFrameFieldNumber); +} + +void VideoFrameModule::ParseTracePacketData(const TracePacket::Decoder& decoder, + int64_t ts, + const TracePacketData&, + uint32_t field_id) { + if (field_id != TracePacket::kVideoFrameFieldNumber) { + return; + } + + protos::pbzero::VideoFrame::Decoder frame(decoder.video_frame()); + + // Insert metadata row. + tables::AndroidVideoFramesTable::Row row; + row.ts = ts; + row.frame_number = + frame.has_frame_number() ? static_cast(frame.frame_number()) : 0; + if (frame.has_track_name()) { + row.track_name = context_->storage->InternString(frame.track_name()); + } + if (frame.has_track_id()) { + row.track_id = frame.track_id(); + } + + auto* table = context_->storage->mutable_video_frames_table(); + uint32_t row_idx = table->Insert(row).row; + + // Store image data — prefer JPEG, fall back to WebP. + protozero::ConstBytes img = {}; + if (frame.has_jpg_image()) { + img = frame.jpg_image(); + } else if (frame.has_webp_image()) { + img = frame.webp_image(); + } + if (img.size > 0) { + auto blob = TraceBlob::Allocate(img.size); + memcpy(blob.data(), img.data, img.size); + auto* blob_vec = context_->storage->mutable_video_frame_data(); + if (blob_vec->size() <= row_idx) { + blob_vec->resize(row_idx + 1); + } + (*blob_vec)[row_idx] = TraceBlobView(std::move(blob), 0, img.size); + } +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/proto/video_frame_module.h b/src/trace_processor/importers/proto/video_frame_module.h new file mode 100644 index 0000000000..2af19d78ad --- /dev/null +++ b/src/trace_processor/importers/proto/video_frame_module.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_VIDEO_FRAME_MODULE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_VIDEO_FRAME_MODULE_H_ + +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/trace_processor/importers/proto/proto_importer_module.h" + +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +// Parses TracePacket.video_frame messages and stores JPEG image data +// in a dedicated blob vector on TraceStorage (not in the args table). +class VideoFrameModule : public ProtoImporterModule { + public: + VideoFrameModule(ProtoImporterModuleContext* module_context, + TraceProcessorContext* context); + ~VideoFrameModule() override; + + void ParseTracePacketData(const protos::pbzero::TracePacket::Decoder& decoder, + int64_t ts, + const TracePacketData& data, + uint32_t field_id) override; + + private: + TraceProcessorContext* context_; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_VIDEO_FRAME_MODULE_H_ diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/BUILD.gn b/src/trace_processor/perfetto_sql/intrinsics/functions/BUILD.gn index fd110b7604..d17e1a53af 100644 --- a/src/trace_processor/perfetto_sql/intrinsics/functions/BUILD.gn +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/BUILD.gn @@ -69,6 +69,8 @@ source_set("functions") { "type_builders.cc", "type_builders.h", "utils.h", + "video_frame_image.cc", + "video_frame_image.h", "window_functions.h", ] deps = [ diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.cc b/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.cc new file mode 100644 index 0000000000..80d98c06e9 --- /dev/null +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.cc @@ -0,0 +1,52 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h" + +#include + +#include "src/trace_processor/sqlite/bindings/sqlite_result.h" +#include "src/trace_processor/sqlite/bindings/sqlite_value.h" + +namespace perfetto::trace_processor { + +void VideoFrameImageFunction::Step(sqlite3_context* ctx, + int, + sqlite3_value** argv) { + auto* storage = sqlite::Function::GetUserData(ctx); + + if (sqlite::value::IsNull(argv[0])) { + sqlite::result::Null(ctx); + return; + } + + int64_t row_id = sqlite::value::Int64(argv[0]); + const auto& blobs = storage->video_frame_data(); + + if (row_id < 0 || static_cast(row_id) >= blobs.size()) { + sqlite::result::Null(ctx); + return; + } + + const auto& blob = blobs[static_cast(row_id)]; + if (blob.data() == nullptr || blob.length() == 0) { + sqlite::result::Null(ctx); + return; + } + + sqlite::result::StaticBytes(ctx, blob.data(), + static_cast(blob.length())); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h b/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h new file mode 100644 index 0000000000..c02af810b0 --- /dev/null +++ b/src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h @@ -0,0 +1,36 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_INTRINSICS_FUNCTIONS_VIDEO_FRAME_IMAGE_H_ +#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_INTRINSICS_FUNCTIONS_VIDEO_FRAME_IMAGE_H_ + +#include "src/trace_processor/sqlite/bindings/sqlite_function.h" +#include "src/trace_processor/storage/trace_storage.h" + +namespace perfetto::trace_processor { + +// video_frame_image(row_id) returns the raw JPEG bytes for a video frame. +// The row_id corresponds to the row number in __intrinsic_video_frames. +struct VideoFrameImageFunction + : public sqlite::Function { + static constexpr char kName[] = "video_frame_image"; + static constexpr int kArgCount = 1; + + using UserData = TraceStorage; + static void Step(sqlite3_context* ctx, int argc, sqlite3_value** argv); +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_INTRINSICS_FUNCTIONS_VIDEO_FRAME_IMAGE_H_ diff --git a/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn index 5acffec133..add0e62e48 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn @@ -66,6 +66,7 @@ perfetto_sql_source_set("android") { "thread.sql", "user_list.sql", "version.sql", + "video_frames.sql", "wakeups.sql", ] } diff --git a/src/trace_processor/perfetto_sql/stdlib/android/video_frames.sql b/src/trace_processor/perfetto_sql/stdlib/android/video_frames.sql new file mode 100644 index 0000000000..8e64b3e3de --- /dev/null +++ b/src/trace_processor/perfetto_sql/stdlib/android/video_frames.sql @@ -0,0 +1,34 @@ +-- Copyright 2026 The Android Open Source Project +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Video frames captured from the device display. +CREATE PERFETTO TABLE android_video_frames ( + -- Row id. Pass to video_frame_image() to get the JPEG bytes. + id ID(__intrinsic_video_frames.id), + -- Timestamp of the frame capture. + ts TIMESTAMP, + -- Sequential frame number within the capture session. + frame_number LONG, + -- Display name for the track in the UI. + track_name STRING, + -- Unique identifier for the stream. + track_id LONG +) AS +SELECT + id, + ts, + frame_number, + track_name, + track_id +FROM __intrinsic_video_frames; diff --git a/src/trace_processor/storage/trace_storage.h b/src/trace_processor/storage/trace_storage.h index 6cb2fb06a6..9e1ace3fc4 100644 --- a/src/trace_processor/storage/trace_storage.h +++ b/src/trace_processor/storage/trace_storage.h @@ -600,6 +600,19 @@ class TraceStorage { return mutable_table(); } + const tables::AndroidVideoFramesTable& video_frames_table() const { + return table(); + } + tables::AndroidVideoFramesTable* mutable_video_frames_table() { + return mutable_table(); + } + const std::vector& video_frame_data() const { + return video_frame_data_; + } + std::vector* mutable_video_frame_data() { + return &video_frame_data_; + } + const tables::AndroidGameInterventionListTable& android_game_intervention_list_table() const { return table(); @@ -1170,6 +1183,9 @@ class TraceStorage { VirtualTrackSlices virtual_track_slices_; SqlStats sql_stats_; + // Video frame image data. Indexed by AndroidVideoFramesTable row id. + std::vector video_frame_data_; + // ETM tables // Indexed by tables::EtmV4ConfigurationTable::Id std::vector> etm_v4_configuration_data_; diff --git a/src/trace_processor/tables/android_tables.py b/src/trace_processor/tables/android_tables.py index 7e5e49fb6c..cebe156af1 100644 --- a/src/trace_processor/tables/android_tables.py +++ b/src/trace_processor/tables/android_tables.py @@ -396,6 +396,33 @@ ), ) +ANDROID_VIDEO_FRAMES_TABLE = Table( + python_module=__file__, + class_name='AndroidVideoFramesTable', + sql_name='__intrinsic_video_frames', + columns=[ + C('ts', + CppInt64(), + cpp_access=CppAccess.READ, + cpp_access_duration=CppAccessDuration.POST_FINALIZATION), + C('frame_number', CppInt64()), + C('track_name', CppOptional(CppString())), + C('track_id', CppOptional(CppUint32())), + ], + tabledoc=TableDoc( + doc=''' + Video frames captured from the device display. + JPEG image data is stored separately and accessed via the + video_frame_image() SQL function. + ''', + group='Android', + columns={ + 'ts': 'Timestamp of the frame capture.', + 'frame_number': 'Sequential frame number within the session.', + 'track_name': 'Display name for the track in the UI.', + 'track_id': 'Unique identifier for the stream.', + })) + # Keep this list sorted. ALL_TABLES = [ ANDROID_AFLAGS_TABLE, @@ -407,4 +434,5 @@ ANDROID_LOG_TABLE, ANDROID_MOTION_EVENTS_TABLE, ANDROID_USER_LIST_TABLE, + ANDROID_VIDEO_FRAMES_TABLE, ] diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index 7383b824e2..0f53dba0ac 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc @@ -114,6 +114,7 @@ #include "src/trace_processor/perfetto_sql/intrinsics/functions/trees/tree_functions.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/type_builders.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/utils.h" +#include "src/trace_processor/perfetto_sql/intrinsics/functions/video_frame_image.h" #include "src/trace_processor/perfetto_sql/intrinsics/functions/window_functions.h" #include "src/trace_processor/perfetto_sql/intrinsics/operators/counter_mipmap_operator.h" #include "src/trace_processor/perfetto_sql/intrinsics/operators/slice_mipmap_operator.h" @@ -442,7 +443,8 @@ uint64_t GetBoundsMutationCount(const TraceStorage& storage) { storage.heap_graph_object_table().mutations() + storage.perf_sample_table().mutations() + storage.instruments_sample_table().mutations() + - storage.cpu_profile_stack_sample_table().mutations(); + storage.cpu_profile_stack_sample_table().mutations() + + storage.video_frames_table().mutations(); } // IMPORTANT: GetBoundsMutationCount and GetTraceTimestampBoundsNs must be kept @@ -488,6 +490,10 @@ std::pair GetTraceTimestampBoundsNs( start_ns = std::min(it.ts(), start_ns); end_ns = std::max(it.ts(), end_ns); } + for (auto it = storage.video_frames_table().IterateRows(); it; ++it) { + start_ns = std::min(it.ts(), start_ns); + end_ns = std::max(it.ts(), end_ns); + } for (auto it = storage.instruments_sample_table().IterateRows(); it; ++it) { start_ns = std::min(it.ts(), start_ns); end_ns = std::max(it.ts(), end_ns); @@ -1118,6 +1124,7 @@ std::vector TraceProcessorImpl::GetStaticTables( AddStaticTable(tables, storage->mutable_android_game_intervenion_list_table()); AddStaticTable(tables, storage->mutable_android_log_table()); + AddStaticTable(tables, storage->mutable_video_frames_table()); AddStaticTable(tables, storage->mutable_build_flags_table()); AddStaticTable(tables, storage->mutable_modules_table()); AddStaticTable(tables, storage->mutable_clock_snapshot_table()); @@ -1323,6 +1330,7 @@ std::unique_ptr TraceProcessorImpl::InitPerfettoSqlEngine( RegisterFunction(engine.get(), storage); RegisterFunction( engine.get(), std::make_unique(storage)); + RegisterFunction(engine.get(), storage); RegisterFunction( engine.get(), std::make_unique(storage)); RegisterFunction(engine.get(), context->clock_converter.get()); diff --git a/test/trace_processor/diff_tests/parser/android/generate_video_trace.py b/test/trace_processor/diff_tests/parser/android/generate_video_trace.py new file mode 100644 index 0000000000..d4d9e06cf1 --- /dev/null +++ b/test/trace_processor/diff_tests/parser/android/generate_video_trace.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generates a test trace with VideoFrame packets. + +Produces two streams ("Front Camera" and "Rear Camera") interleaved by +timestamp, each with colored JPEG frames suitable for filmstrip testing. + +Usage: + python3 generate_video_trace.py [output.pb] [frames_per_stream] [fps] +""" + +import io +import subprocess +import sys + + +def encode_varint(value): + b = io.BytesIO() + while value > 0x7F: + b.write(bytes([0x80 | (value & 0x7F)])) + value >>= 7 + b.write(bytes([value & 0x7F])) + return b.getvalue() + + +def varint_field(field_num, value): + tag = (field_num << 3) | 0 + return encode_varint(tag) + encode_varint(value) + + +def bytes_field(field_num, data): + if isinstance(data, str): + data = data.encode('utf-8') + tag = (field_num << 3) | 2 + return encode_varint(tag) + encode_varint(len(data)) + data + + +def make_jpeg(frame_num, total, label, hue_offset): + hue = (int(frame_num / max(total, 1) * 360) + hue_offset) % 360 + r = subprocess.run( + [ + 'convert', + '-size', + '320x180', + 'xc:hsl(%d,60%%,90%%)' % hue, + '-gravity', + 'center', + '-pointsize', + '48', + '-fill', + 'white', + '-annotate', + '0', + str(frame_num), + '-gravity', + 'south', + '-pointsize', + '14', + '-annotate', + '0', + '%s - Frame %d' % (label, frame_num), + 'jpeg:-', + ], + capture_output=True, + timeout=10, + ) + if r.returncode != 0: + raise RuntimeError('ImageMagick convert failed: ' + r.stderr.decode()) + return r.stdout + + +def make_video_frame(frame_num, jpeg, track_name, track_id): + """Build a VideoFrame proto message.""" + msg = varint_field(1, frame_num) # frame_number + msg += bytes_field(2, jpeg) # jpg_image + msg += bytes_field(3, track_name) # track_name + msg += varint_field(4, track_id) # track_id + return msg + + +def make_trace_packet(ts, video_frame_bytes): + """Build a TracePacket with timestamp + video_frame.""" + pkt = varint_field(8, ts) # timestamp + pkt += varint_field(58, 6) # timestamp_clock_id = BOOTTIME + pkt += bytes_field(129, video_frame_bytes) # video_frame + return bytes_field(1, pkt) # Trace.packet + + +def main(): + output = sys.argv[1] if len(sys.argv) > 1 else '/tmp/video_trace.pb' + frames_per_stream = int(sys.argv[2]) if len(sys.argv) > 2 else 30 + fps = int(sys.argv[3]) if len(sys.argv) > 3 else 30 + interval_ns = 1_000_000_000 // fps + start_ts = 1_000_000_000 + + streams = [ + { + 'track_id': 1, + 'track_name': 'Front Camera', + 'hue_offset': 0 + }, + { + 'track_id': 2, + 'track_name': 'Rear Camera', + 'hue_offset': 180 + }, + ] + + # Build all frames with timestamps, then sort by ts for interleaving. + events = [] + for s in streams: + for i in range(frames_per_stream): + ts = start_ts + i * interval_ns + events.append((ts, i, s)) + + events.sort(key=lambda e: (e[0], e[2]['track_id'])) + + trace = io.BytesIO() + for ts, frame_num, s in events: + jpeg = make_jpeg(frame_num, frames_per_stream, s['track_name'], + s['hue_offset']) + vf = make_video_frame(frame_num, jpeg, s['track_name'], s['track_id']) + trace.write(make_trace_packet(ts, vf)) + if frame_num % 10 == 0 and s['track_id'] == streams[0]['track_id']: + print('Frame %d/%d per stream (%d bytes)' % + (frame_num + 1, frames_per_stream, len(jpeg))) + + data = trace.getvalue() + with open(output, 'wb') as f: + f.write(data) + total = len(streams) * frames_per_stream + print('Wrote %d frames (%d streams x %d) = %d bytes to %s' % + (total, len(streams), frames_per_stream, len(data), output)) + + +if __name__ == '__main__': + main() diff --git a/test/trace_processor/diff_tests/parser/android/tests.py b/test/trace_processor/diff_tests/parser/android/tests.py index 8f3ae5f5cd..00841a89c3 100644 --- a/test/trace_processor/diff_tests/parser/android/tests.py +++ b/test/trace_processor/diff_tests/parser/android/tests.py @@ -290,3 +290,145 @@ def test_android_system_property_counter_machine_id(self): "ScreenState",0,1000,2.000000 "ScreenState",1,2000,1.000000 """)) + + def test_video_frame_basic(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + timestamp: 1000000000 + timestamp_clock_id: 6 + video_frame { + frame_number: 0 + track_name: "Front Camera" + track_id: 1 + } + } + packet { + timestamp: 1033333333 + timestamp_clock_id: 6 + video_frame { + frame_number: 1 + track_name: "Front Camera" + track_id: 1 + } + } + packet { + timestamp: 1066666666 + timestamp_clock_id: 6 + video_frame { + frame_number: 2 + track_name: "Rear Camera" + track_id: 2 + } + } + """), + query=""" + INCLUDE PERFETTO MODULE android.video_frames; + SELECT ts, frame_number, track_name, track_id + FROM android_video_frames + ORDER BY ts; + """, + out=Csv(""" + "ts","frame_number","track_name","track_id" + 1000000000,0,"Front Camera",1 + 1033333333,1,"Front Camera",1 + 1066666666,2,"Rear Camera",2 + """)) + + def test_video_frame_no_track_fields(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + timestamp: 1000000000 + timestamp_clock_id: 6 + video_frame { + frame_number: 0 + } + } + packet { + timestamp: 1033333333 + timestamp_clock_id: 6 + video_frame { + frame_number: 1 + } + } + """), + query=""" + INCLUDE PERFETTO MODULE android.video_frames; + SELECT ts, frame_number, track_name, track_id + FROM android_video_frames + ORDER BY ts; + """, + out=Csv(""" + "ts","frame_number","track_name","track_id" + 1000000000,0,"[NULL]","[NULL]" + 1033333333,1,"[NULL]","[NULL]" + """)) + + def test_video_frame_trace_bounds(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + timestamp: 1000000000 + timestamp_clock_id: 6 + video_frame { + frame_number: 0 + track_id: 1 + } + } + packet { + timestamp: 2000000000 + timestamp_clock_id: 6 + video_frame { + frame_number: 1 + track_id: 1 + } + } + """), + query=""" + SELECT trace_start() AS s, trace_end() AS e, trace_dur() AS d; + """, + out=Csv(""" + "s","e","d" + 1000000000,2000000000,1000000000 + """)) + + def test_video_frame_multi_stream_grouping(self): + return DiffTestBlueprint( + trace=TextProto(r""" + packet { + timestamp: 1000000000 + timestamp_clock_id: 6 + video_frame { frame_number: 0 track_name: "A" track_id: 1 } + } + packet { + timestamp: 1000000000 + timestamp_clock_id: 6 + video_frame { frame_number: 0 track_name: "B" track_id: 2 } + } + packet { + timestamp: 1033333333 + timestamp_clock_id: 6 + video_frame { frame_number: 1 track_name: "A" track_id: 1 } + } + packet { + timestamp: 1033333333 + timestamp_clock_id: 6 + video_frame { frame_number: 1 track_name: "B" track_id: 2 } + } + """), + query=""" + INCLUDE PERFETTO MODULE android.video_frames; + SELECT + COALESCE(track_id, 0) AS tid, + COALESCE(track_name, 'Video Frames') AS tname, + COUNT(*) AS cnt + FROM android_video_frames + GROUP BY tid + ORDER BY tid; + """, + out=Csv(""" + "tid","tname","cnt" + 1,"A",2 + 2,"B",2 + """)) diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/index.ts b/ui/src/plugins/dev.perfetto.VideoFrames/index.ts new file mode 100644 index 0000000000..d6cd8b84ef --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/index.ts @@ -0,0 +1,87 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {TrackNode} from '../../public/workspace'; +import {NUM, STR} from '../../trace_processor/query_result'; +import {Trace} from '../../public/trace'; +import {PerfettoPlugin} from '../../public/plugin'; +import {createVideoFramesTrack} from './video_frames_track'; +import {VideoFramesSelectionTab} from './video_frames_selection_tab'; +import {VideoFramePlayer} from './playback_state'; + +interface StreamInfo { + trackId: number; + trackName: string; +} + +export default class implements PerfettoPlugin { + static readonly id = 'dev.perfetto.VideoFrames'; + + async onTraceLoad(ctx: Trace): Promise { + await ctx.engine.query('INCLUDE PERFETTO MODULE android.video_frames'); + + const res = await ctx.engine.query(` + SELECT + COALESCE(track_id, 0) AS trackId, + COALESCE(track_name, 'Video Frames') AS trackName + FROM android_video_frames + GROUP BY trackId + ORDER BY trackId + `); + + const streams: StreamInfo[] = []; + const it = res.iter({trackId: NUM, trackName: STR}); + for (; it.valid(); it.next()) { + streams.push({trackId: it.trackId, trackName: it.trackName}); + } + if (streams.length === 0) return; + + const group = new TrackNode({ + name: 'Video Frames', + isSummary: true, + sortOrder: -55, + }); + + for (const stream of streams) { + const uri = `/video_frames/${stream.trackId}`; + const viewName = `_video_frames_track_${stream.trackId}`; + + await ctx.engine.query(` + CREATE OR REPLACE PERFETTO VIEW ${viewName} AS + SELECT + id, + ts, + 0 AS dur, + 'Frame ' || frame_number AS name + FROM android_video_frames + WHERE COALESCE(track_id, 0) = ${stream.trackId} + `); + + const player = new VideoFramePlayer(ctx, uri, stream.trackId); + + ctx.tracks.registerTrack({ + uri, + renderer: createVideoFramesTrack(ctx, uri, viewName, player), + }); + + group.addChildInOrder(new TrackNode({uri, name: stream.trackName})); + + ctx.selection.registerAreaSelectionTab( + new VideoFramesSelectionTab(ctx, uri, stream.trackId, stream.trackName), + ); + } + + ctx.defaultWorkspace.addChildInOrder(group); + } +} diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/playback_state.ts b/ui/src/plugins/dev.perfetto.VideoFrames/playback_state.ts new file mode 100644 index 0000000000..8d5154736e --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/playback_state.ts @@ -0,0 +1,172 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import m from 'mithril'; +import {Trace} from '../../public/trace'; +import {BLOB, LONG, NUM} from '../../trace_processor/query_result'; + +export interface FrameInfo { + id: number; + ts: bigint; + frameNumber: number; +} + +export const FPS_OPTIONS = [1, 5, 10, 15, 30, 60, 120, 240]; + +// Shared fps across all players in the session. +let sessionFps = 30; + +export function getSessionFps(): number { + return sessionFps; +} +export function setSessionFps(fps: number): void { + sessionFps = fps; +} + +// Player for a single video frame stream (identified by trackId). +// One instance per stream, created once in onTraceLoad and reused across +// panel recreations. +export class VideoFramePlayer { + readonly trace: Trace; + readonly trackUri: string; + private readonly trackId: number; + + frames: FrameInfo[] = []; + currentIdx = 0; + imageUrl?: string; + playing = false; + + private playTimer?: ReturnType; + private framesLoaded = false; + private playbackStartIdx = 0; + + constructor(trace: Trace, trackUri: string, trackId: number) { + this.trace = trace; + this.trackUri = trackUri; + this.trackId = trackId; + } + + get fps(): number { + return sessionFps; + } + + get currentFrame(): FrameInfo | undefined { + return this.frames[this.currentIdx]; + } + + async ensureFramesLoaded(): Promise { + if (this.framesLoaded) return; + const res = await this.trace.engine.query(` + SELECT id, ts, frame_number AS frameNumber + FROM android_video_frames + WHERE COALESCE(track_id, 0) = ${this.trackId} + ORDER BY ts + `); + const it = res.iter({id: NUM, ts: LONG, frameNumber: NUM}); + this.frames = []; + for (; it.valid(); it.next()) { + this.frames.push({id: it.id, ts: it.ts, frameNumber: it.frameNumber}); + } + this.framesLoaded = true; + } + + async goToId(eventId: number): Promise { + const idx = this.frames.findIndex((f) => f.id === eventId); + if (idx >= 0) { + await this.loadImage(idx); + } + } + + async loadImage(idx: number): Promise { + if (idx < 0 || idx >= this.frames.length) return; + this.currentIdx = idx; + + if (this.imageUrl) { + URL.revokeObjectURL(this.imageUrl); + this.imageUrl = undefined; + } + + const id = this.frames[idx].id; + const res = await this.trace.engine.query( + `SELECT video_frame_image(${id}) AS img`, + ); + const row = res.firstRow({img: BLOB}); + if (row.img.length > 0) { + const blob = new Blob([row.img]); + this.imageUrl = URL.createObjectURL(blob); + } + m.redraw(); + } + + togglePlay(): void { + if (this.playing) { + this.stop(); + } else { + this.play(); + } + m.redraw(); + } + + play(): void { + this.stop(); + this.playing = true; + this.playbackStartIdx = this.currentIdx; + this.playTimer = setInterval(() => { + if (this.currentIdx < this.frames.length - 1) { + const nextIdx = this.currentIdx + 1; + this.loadImage(nextIdx).then(() => { + if (this.playing) { + this.trace.selection.selectTrackEvent( + this.trackUri, + this.frames[nextIdx].id, + ); + } + }); + } else { + this.stop(); + this.loadImage(this.playbackStartIdx).then(() => { + this.trace.selection.selectTrackEvent( + this.trackUri, + this.frames[this.playbackStartIdx].id, + ); + }); + } + }, 1000 / sessionFps); + } + + stop(): void { + this.playing = false; + if (this.playTimer !== undefined) { + clearInterval(this.playTimer); + this.playTimer = undefined; + } + } + + prev(): void { + if (this.currentIdx > 0) this.loadImage(this.currentIdx - 1); + } + + next(): void { + if (this.currentIdx < this.frames.length - 1) { + this.loadImage(this.currentIdx + 1); + } + } + + setFps(fps: number): void { + sessionFps = fps; + if (this.playing) { + this.play(); + } + } +} diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/styles.scss b/ui/src/plugins/dev.perfetto.VideoFrames/styles.scss new file mode 100644 index 0000000000..3e204a78d7 --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/styles.scss @@ -0,0 +1,15 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@import "video_panel"; diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/video_frame_details_panel.ts b/ui/src/plugins/dev.perfetto.VideoFrames/video_frame_details_panel.ts new file mode 100644 index 0000000000..61cbcb0b10 --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/video_frame_details_panel.ts @@ -0,0 +1,128 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import m from 'mithril'; +import {TrackEventDetailsPanel} from '../../public/details_panel'; +import {TrackEventSelection} from '../../public/selection'; +import {DetailsShell} from '../../widgets/details_shell'; +import {GridLayout} from '../../widgets/grid_layout'; +import {Section} from '../../widgets/section'; +import {Tree, TreeNode} from '../../widgets/tree'; +import {Button, ButtonBar} from '../../widgets/button'; +import {Intent} from '../../widgets/common'; +import {Select} from '../../widgets/select'; +import {Timestamp} from '../../components/widgets/timestamp'; +import {Time} from '../../base/time'; +import {VideoFramePlayer, FPS_OPTIONS} from './playback_state'; + +export class VideoFrameDetailsPanel implements TrackEventDetailsPanel { + private readonly player: VideoFramePlayer; + + constructor(player: VideoFramePlayer) { + this.player = player; + } + + async load(sel: TrackEventSelection) { + // If playback is driving the selection, the player already has the + // right frame loaded — nothing to do. + if (this.player.playing) return; + + await this.player.ensureFramesLoaded(); + await this.player.goToId(sel.eventId); + } + + render() { + const p = this.player; + const frame = p.currentFrame; + if (!frame) { + return m(DetailsShell, {title: 'Video Frame'}, m('span', 'Loading...')); + } + + return m( + DetailsShell, + { + title: 'Video Frame', + description: `Frame ${frame.frameNumber}`, + buttons: this.renderControls(), + }, + m( + GridLayout, + m( + Section, + {title: 'Details'}, + m( + Tree, + m(TreeNode, { + left: 'Frame number', + right: `${frame.frameNumber}`, + }), + m(TreeNode, { + left: 'Timestamp', + right: m(Timestamp, { + trace: p.trace, + ts: Time.fromRaw(frame.ts), + }), + }), + ), + ), + m( + Section, + {title: 'Preview'}, + p.imageUrl + ? m('img.pf-video-frame-preview', {src: p.imageUrl}) + : m('span', 'No image data'), + ), + ), + ); + } + + private renderControls(): m.Children { + const p = this.player; + const idx = p.currentIdx; + const total = p.frames.length; + + return m(ButtonBar, [ + m(Button, { + icon: 'skip_previous', + compact: true, + disabled: idx <= 0 || p.playing, + onclick: () => p.prev(), + }), + m(Button, { + icon: p.playing ? 'pause' : 'play_arrow', + intent: p.playing ? Intent.Warning : Intent.Primary, + compact: true, + onclick: () => p.togglePlay(), + }), + m(Button, { + icon: 'skip_next', + compact: true, + disabled: idx >= total - 1 || p.playing, + onclick: () => p.next(), + }), + m( + Select, + { + value: String(p.fps), + onchange: (e: Event) => { + p.setFps(Number((e.target as HTMLSelectElement).value)); + }, + }, + FPS_OPTIONS.map((f) => + m('option', {value: String(f), selected: f === p.fps}, `${f} fps`), + ), + ), + ]); + } +} diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_selection_tab.ts b/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_selection_tab.ts new file mode 100644 index 0000000000..e4e4daa76c --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_selection_tab.ts @@ -0,0 +1,250 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import m from 'mithril'; +import {Trace} from '../../public/trace'; +import { + AreaSelection, + AreaSelectionTab, + ContentWithLoadingFlag, +} from '../../public/selection'; +import {BLOB, LONG, NUM} from '../../trace_processor/query_result'; +import {Button, ButtonBar} from '../../widgets/button'; +import {Intent} from '../../widgets/common'; +import {Select} from '../../widgets/select'; +import {GridLayout} from '../../widgets/grid_layout'; +import {Section} from '../../widgets/section'; +import {Tree, TreeNode} from '../../widgets/tree'; +import {Timestamp} from '../../components/widgets/timestamp'; +import {Time} from '../../base/time'; +import { + FrameInfo, + FPS_OPTIONS, + getSessionFps, + setSessionFps, +} from './playback_state'; + +export class VideoFramesSelectionTab implements AreaSelectionTab { + readonly id: string; + readonly name: string; + readonly priority = 10; + + private readonly trace: Trace; + private readonly trackUri: string; + private readonly trackId: number; + private frames: FrameInfo[] = []; + private currentIdx = 0; + private imageUrl?: string; + private playing = false; + private playTimer?: ReturnType; + private loading = false; + private lastSelectionKey = ''; + private playbackStartIdx = 0; + + constructor( + trace: Trace, + trackUri: string, + trackId: number, + trackName: string, + ) { + this.trace = trace; + this.trackUri = trackUri; + this.trackId = trackId; + this.id = `video_frames_playback_${trackId}`; + this.name = trackName; + } + + render(selection: AreaSelection): ContentWithLoadingFlag | undefined { + if (!selection.trackUris.includes(this.trackUri)) return undefined; + + const key = `${selection.start}-${selection.end}`; + if (key !== this.lastSelectionKey) { + this.lastSelectionKey = key; + this.stop(); + this.loadFramesForSelection(selection); + } + + if (this.frames.length === 0 && !this.loading) { + return undefined; + } + + const idx = this.currentIdx; + const total = this.frames.length; + const frame = this.frames[idx]; + + return { + isLoading: this.loading, + buttons: this.renderControls(idx, total), + content: m( + GridLayout, + m( + Section, + {title: 'Details'}, + frame !== undefined + ? m( + Tree, + m(TreeNode, { + left: 'Frame', + right: `${idx + 1} of ${total}`, + }), + m(TreeNode, { + left: 'Timestamp', + right: m(Timestamp, { + trace: this.trace, + ts: Time.fromRaw(frame.ts), + }), + }), + ) + : m('span', 'Loading...'), + ), + m( + Section, + {title: 'Preview'}, + this.imageUrl + ? m('img.pf-video-frame-preview', {src: this.imageUrl}) + : m('span', this.loading ? '' : 'No image'), + ), + ), + }; + } + + private renderControls(idx: number, total: number): m.Children { + return m(ButtonBar, [ + m(Button, { + icon: 'skip_previous', + compact: true, + disabled: idx <= 0 || this.playing, + onclick: () => this.prev(), + }), + m(Button, { + icon: this.playing ? 'pause' : 'play_arrow', + intent: this.playing ? Intent.Warning : Intent.Primary, + compact: true, + onclick: () => this.togglePlay(), + }), + m(Button, { + icon: 'skip_next', + compact: true, + disabled: idx >= total - 1 || this.playing, + onclick: () => this.next(), + }), + m( + Select, + { + value: String(getSessionFps()), + onchange: (e: Event) => { + setSessionFps(Number((e.target as HTMLSelectElement).value)); + if (this.playing) { + this.play(); + } + }, + }, + FPS_OPTIONS.map((f) => + m( + 'option', + {value: String(f), selected: f === getSessionFps()}, + `${f} fps`, + ), + ), + ), + ]); + } + + private async loadFramesForSelection(selection: AreaSelection) { + this.loading = true; + this.frames = []; + this.currentIdx = 0; + m.redraw(); + + const res = await this.trace.engine.query(` + SELECT id, ts, frame_number AS frameNumber + FROM android_video_frames + WHERE COALESCE(track_id, 0) = ${this.trackId} + AND ts >= ${selection.start} AND ts <= ${selection.end} + ORDER BY ts + `); + const it = res.iter({id: NUM, ts: LONG, frameNumber: NUM}); + for (; it.valid(); it.next()) { + this.frames.push({id: it.id, ts: it.ts, frameNumber: it.frameNumber}); + } + this.loading = false; + + if (this.frames.length > 0) { + await this.loadImage(0); + } + m.redraw(); + } + + private async loadImage(idx: number) { + if (idx < 0 || idx >= this.frames.length) return; + this.currentIdx = idx; + + if (this.imageUrl) { + URL.revokeObjectURL(this.imageUrl); + this.imageUrl = undefined; + } + + const id = this.frames[idx].id; + const res = await this.trace.engine.query( + `SELECT video_frame_image(${id}) AS img`, + ); + const row = res.firstRow({img: BLOB}); + if (row.img.length > 0) { + const blob = new Blob([row.img]); + this.imageUrl = URL.createObjectURL(blob); + } + m.redraw(); + } + + private prev() { + if (this.currentIdx > 0) this.loadImage(this.currentIdx - 1); + } + + private next() { + if (this.currentIdx < this.frames.length - 1) { + this.loadImage(this.currentIdx + 1); + } + } + + private togglePlay() { + if (this.playing) { + this.stop(); + } else { + this.play(); + } + m.redraw(); + } + + private play() { + this.stop(); + this.playing = true; + this.playbackStartIdx = this.currentIdx; + this.playTimer = setInterval(() => { + if (this.currentIdx < this.frames.length - 1) { + this.loadImage(this.currentIdx + 1); + } else { + this.stop(); + this.loadImage(this.playbackStartIdx); + } + }, 1000 / getSessionFps()); + } + + private stop() { + this.playing = false; + if (this.playTimer !== undefined) { + clearInterval(this.playTimer); + this.playTimer = undefined; + } + } +} diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_track.ts b/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_track.ts new file mode 100644 index 0000000000..fdf7adb1cd --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/video_frames_track.ts @@ -0,0 +1,42 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {SliceTrack} from '../../components/tracks/slice_track'; +import {Trace} from '../../public/trace'; +import {SourceDataset} from '../../trace_processor/dataset'; +import {LONG, NUM, STR} from '../../trace_processor/query_result'; +import {VideoFrameDetailsPanel} from './video_frame_details_panel'; +import {VideoFramePlayer} from './playback_state'; + +export function createVideoFramesTrack( + trace: Trace, + uri: string, + viewName: string, + player: VideoFramePlayer, +) { + return SliceTrack.create({ + trace, + uri, + dataset: new SourceDataset({ + schema: { + id: NUM, + ts: LONG, + dur: LONG, + name: STR, + }, + src: viewName, + }), + detailsPanel: () => new VideoFrameDetailsPanel(player), + }); +} diff --git a/ui/src/plugins/dev.perfetto.VideoFrames/video_panel.scss b/ui/src/plugins/dev.perfetto.VideoFrames/video_panel.scss new file mode 100644 index 0000000000..a9658bb45d --- /dev/null +++ b/ui/src/plugins/dev.perfetto.VideoFrames/video_panel.scss @@ -0,0 +1,21 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +.pf-video-frame-preview { + display: block; + max-width: 100%; + max-height: 100%; + border: 1px solid var(--md-sys-color-outline-variant); + border-radius: 4px; +}