Skip to content

Commit 9cd105d

Browse files
blakebyrnesblakebyrnes
committed
fix(profile): resolve stalled dom collector
1 parent 72a201f commit 9cd105d

File tree

5 files changed

+164
-9
lines changed

5 files changed

+164
-9
lines changed

‎browser-profiler/main/lib/local-tooling/ChromeUtils.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,35 @@ export async function navigateDevtoolsToUrl(
106106
try {
107107
client = await connect(developerToolsPort);
108108
console.log('Connected to chrome devtools (%s)', developerToolsPort);
109-
const { Network, Page } = client;
109+
const { Network, Page, Runtime, Log } = client;
110110
await Network.enable();
111+
const shouldLogConsole =
112+
process.env.DOM_CONSOLE === '1' || process.env.DOM_EXTRACTOR_DEBUG === '1';
113+
if (shouldLogConsole) {
114+
await Runtime.enable();
115+
await Log.enable();
116+
Runtime.consoleAPICalled(event => {
117+
const args = (event.args || [])
118+
.map(x => x.value ?? x.description ?? x.type)
119+
.join(' ');
120+
console.log(`[chrome-console] ${event.type}: ${args}`);
121+
});
122+
Runtime.exceptionThrown(event => {
123+
const details = event.exceptionDetails;
124+
const desc = details.exception?.description ?? details.text;
125+
console.log(`[chrome-exception] ${desc}`);
126+
});
127+
Log.entryAdded(event => {
128+
console.log(`[chrome-log] ${event.entry.level}: ${event.entry.text}`);
129+
});
130+
}
111131
await Page.enable();
112132
await Page.navigate({ url });
133+
// Close the client after page load (or timeout) to avoid leaking connections.
134+
await Promise.race([
135+
Page.loadEventFired(),
136+
new Promise<void>(resolve => setTimeout(resolve, 5_000)),
137+
]);
113138
} catch (err) {
114139
console.error('DEVTOOLS ERROR: ', err);
115140
} finally {

‎double-agent/collect/plugins/browser-dom-environment/index.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export default class BrowserDomPlugin extends Plugin {
1818
public initialize() {
1919
this.registerRoute('allHttp1', '/', this.loadScript);
2020
this.registerRoute('allHttp1', '/save', this.save);
21+
this.registerRoute('allHttp1', '/dom-extractor-stall', this.handleStallLog);
2122
this.registerRoute('allHttp1', '/load-dedicated-worker', this.loadDedicatedWorker);
2223
this.registerRoute('allHttp1', '/dedicated-worker.js', dedicatedWorkerScript);
2324
this.registerRoute('allHttp1', '/load-service-worker', this.loadServiceWorker);
@@ -77,6 +78,8 @@ export default class BrowserDomPlugin extends Plugin {
7778
pageUrl: ctx.url.href,
7879
pageHost: ctx.url.host,
7980
pageName: PageNames.BrowserDom,
81+
debugToConsole: process.env.DOM_CONSOLE === '1',
82+
stallLogUrl: ctx.buildUrl('/dom-extractor-stall'),
8083
};
8184
document.injectBodyTag(`
8285
<script type="text/javascript">
@@ -168,6 +171,19 @@ export default class BrowserDomPlugin extends Plugin {
168171
ctx.res.end();
169172
}
170173

174+
private async handleStallLog(ctx: IRequestContext) {
175+
const pageName = ctx.req.headers['page-name'] || PageNames.BrowserDom;
176+
const pendingKey = this.getPendingKey(ctx, pageName as string);
177+
const body = ctx.requestDetails.bodyJson as { path?: string; pageName?: string };
178+
const path = body?.path ? String(body.path) : '';
179+
if (path) {
180+
console.warn('[dom-extractor-stall]', pendingKey, ctx.url.href, path);
181+
} else {
182+
console.warn('[dom-extractor-stall]', pendingKey, ctx.url.href);
183+
}
184+
ctx.res.end();
185+
}
186+
171187
private async waitUntilFinished(ctx: IRequestContext) {
172188
const pendingKey = this.getPendingKey(
173189
ctx,

‎double-agent/collect/plugins/browser-dom-environment/injected-scripts/DomExtractor.js‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
function DomExtractor(selfName, pageMeta = {}) {
2-
const { saveToUrl, pageUrl, pageHost, pageName } = pageMeta;
2+
const { saveToUrl, pageUrl, pageHost, pageName, debugToConsole, stallLogUrl } = pageMeta;
33
const skipProps = [
44
'Fingerprint2',
55
'pageQueue',
66
'DomExtractor',
77
'pageLoaded',
88
'axios',
99
'justAFunction',
10+
'append',
11+
'setHTMLUnsafe',
1012
];
11-
const skipValues = ['innerHTML', 'outerHTML', 'innerText', 'outerText'];
13+
const skipValues = ['innerHTML', 'outerHTML', 'innerText', 'outerText', 'epochNanoseconds'];
1214
const doNotInvoke = [
1315
'replaceChildren',
1416
'print',
@@ -31,6 +33,9 @@ function DomExtractor(selfName, pageMeta = {}) {
3133
'writeln',
3234
'replaceWith',
3335
'remove',
36+
'SVGAnimatedInteger',
37+
'createOffer',
38+
'createAnswer',
3439
'self.history.back',
3540
'self.history.forward',
3641
'self.history.go',
@@ -43,6 +48,7 @@ function DomExtractor(selfName, pageMeta = {}) {
4348
'self.navigation.replaceState',
4449
'getUserMedia',
4550
'requestFullscreen',
51+
'requestPointerLock',
4652
'webkitRequestFullScreen',
4753
'webkitRequestFullscreen',
4854
'getDisplayMedia',
@@ -56,6 +62,43 @@ function DomExtractor(selfName, pageMeta = {}) {
5662
`self.XRRigidTransform.new().inverse`,
5763
].map(x => x.replace(/self\./g, `${selfName}.`));
5864
const excludedInheritedKeys = ['name', 'length', 'constructor'];
65+
let lastDebugPath;
66+
let lastConsoleAt = 0;
67+
let stallTimer;
68+
let stallPostedForPath;
69+
const stallTimeoutMs = 5 * 60 * 1000;
70+
let debugCount = 0;
71+
function debugPath(path) {
72+
if (debugToConsole) {
73+
debugCount += 1;
74+
const now = Date.now();
75+
if (debugCount % 500 === 0 || now - lastConsoleAt > 1000) {
76+
lastConsoleAt = now;
77+
try {
78+
console.log('[dom-extractor]', path);
79+
} catch (err) {}
80+
}
81+
}
82+
if (path === lastDebugPath) return;
83+
lastDebugPath = path;
84+
stallPostedForPath = undefined;
85+
if (!stallLogUrl) return;
86+
if (stallTimer) clearTimeout(stallTimer);
87+
stallTimer = setTimeout(() => {
88+
if (!lastDebugPath || stallPostedForPath === lastDebugPath) return;
89+
stallPostedForPath = lastDebugPath;
90+
try {
91+
fetch(stallLogUrl, {
92+
method: 'POST',
93+
body: JSON.stringify({ path: lastDebugPath, pageName }),
94+
headers: {
95+
'Content-Type': 'application/json',
96+
'Page-Name': pageName,
97+
},
98+
}).catch(() => null);
99+
} catch (err) {}
100+
}, stallTimeoutMs);
101+
}
59102
const loadedObjectsRef = new Map([[self, selfName]]);
60103
const loadedObjectsProp = new Map();
61104
const hierarchyNav = new Map();
@@ -99,7 +142,11 @@ function DomExtractor(selfName, pageMeta = {}) {
99142
return newObj;
100143
}
101144
const isNewObject = parentPath.includes('.new()');
102-
if (isNewObject && newObj._$protos[0] === 'HTMLDocument.prototype') {
145+
if (
146+
isNewObject &&
147+
(newObj._$protos[0] === 'HTMLDocument.prototype' ||
148+
newObj._$protos[0] === 'Document.prototype')
149+
) {
103150
newObj._$skipped = 'SKIPPED DOCUMENT';
104151
newObj._$type = 'HTMLDocument.prototype';
105152
return newObj;
@@ -234,6 +281,7 @@ function DomExtractor(selfName, pageMeta = {}) {
234281
if (obj === null || obj === undefined || !key) {
235282
return undefined;
236283
}
284+
debugPath(path);
237285
let accessException;
238286
const value = await new Promise(async (resolve, reject) => {
239287
let didResolve = false;
@@ -361,7 +409,15 @@ function DomExtractor(selfName, pageMeta = {}) {
361409
func = err.toString();
362410
}
363411
try {
364-
if (!doNotInvoke.includes(key) && !doNotInvoke.includes(path) && !value.prototype) {
412+
const isSvgPrototypeMethod =
413+
path.startsWith(`${selfName}.SVG`) && path.includes('.prototype.');
414+
if (
415+
!isSvgPrototypeMethod &&
416+
!doNotInvoke.includes(key) &&
417+
!doNotInvoke.includes(path) &&
418+
!value.prototype
419+
) {
420+
debugPath(`${path}()`);
365421
invocation = await new Promise(async (resolve, reject) => {
366422
const c = setTimeout(() => reject('Promise-like'), 650);
367423
let didReply = false;

‎double-agent/collect/plugins/browser-dom-environment/lib/loadDomExtractorScript.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ export interface IDomExtractorPageMeta {
1010
pageUrl: string;
1111
pageHost: string;
1212
pageName: keyof typeof PageNames | string;
13+
debugToConsole?: boolean;
14+
stallLogUrl?: string;
1315
}

‎plugins/default-browser-emulator/test/DomExtractor.js‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
function DomExtractor(selfName, pageMeta = {}) {
2-
const { saveToUrl, pageUrl, pageHost, pageName } = pageMeta;
2+
const { saveToUrl, pageUrl, pageHost, pageName, debugToConsole, stallLogUrl } = pageMeta;
33
const skipProps = [
44
'Fingerprint2',
55
'pageQueue',
66
'DomExtractor',
77
'pageLoaded',
88
'axios',
99
'justAFunction',
10+
'append',
11+
'setHTMLUnsafe',
1012
];
11-
const skipValues = ['innerHTML', 'outerHTML', 'innerText', 'outerText'];
13+
const skipValues = ['innerHTML', 'outerHTML', 'innerText', 'outerText', 'epochNanoseconds'];
1214
const doNotInvoke = [
1315
'replaceChildren',
1416
'print',
@@ -31,6 +33,9 @@ function DomExtractor(selfName, pageMeta = {}) {
3133
'writeln',
3234
'replaceWith',
3335
'remove',
36+
'SVGAnimatedInteger',
37+
'createOffer',
38+
'createAnswer',
3439
'self.history.back',
3540
'self.history.forward',
3641
'self.history.go',
@@ -43,6 +48,7 @@ function DomExtractor(selfName, pageMeta = {}) {
4348
'self.navigation.replaceState',
4449
'getUserMedia',
4550
'requestFullscreen',
51+
'requestPointerLock',
4652
'webkitRequestFullScreen',
4753
'webkitRequestFullscreen',
4854
'getDisplayMedia',
@@ -56,6 +62,43 @@ function DomExtractor(selfName, pageMeta = {}) {
5662
`self.XRRigidTransform.new().inverse`,
5763
].map(x => x.replace(/self\./g, `${selfName}.`));
5864
const excludedInheritedKeys = ['name', 'length', 'constructor'];
65+
let lastDebugPath;
66+
let lastConsoleAt = 0;
67+
let stallTimer;
68+
let stallPostedForPath;
69+
const stallTimeoutMs = 5 * 60 * 1000;
70+
let debugCount = 0;
71+
function debugPath(path) {
72+
if (debugToConsole) {
73+
debugCount += 1;
74+
const now = Date.now();
75+
if (debugCount % 500 === 0 || now - lastConsoleAt > 1000) {
76+
lastConsoleAt = now;
77+
try {
78+
console.log('[dom-extractor]', path);
79+
} catch (err) {}
80+
}
81+
}
82+
if (path === lastDebugPath) return;
83+
lastDebugPath = path;
84+
stallPostedForPath = undefined;
85+
if (!stallLogUrl) return;
86+
if (stallTimer) clearTimeout(stallTimer);
87+
stallTimer = setTimeout(() => {
88+
if (!lastDebugPath || stallPostedForPath === lastDebugPath) return;
89+
stallPostedForPath = lastDebugPath;
90+
try {
91+
fetch(stallLogUrl, {
92+
method: 'POST',
93+
body: JSON.stringify({ path: lastDebugPath, pageName }),
94+
headers: {
95+
'Content-Type': 'application/json',
96+
'Page-Name': pageName,
97+
},
98+
}).catch(() => null);
99+
} catch (err) {}
100+
}, stallTimeoutMs);
101+
}
59102
const loadedObjectsRef = new Map([[self, selfName]]);
60103
const loadedObjectsProp = new Map();
61104
const hierarchyNav = new Map();
@@ -99,7 +142,11 @@ function DomExtractor(selfName, pageMeta = {}) {
99142
return newObj;
100143
}
101144
const isNewObject = parentPath.includes('.new()');
102-
if (isNewObject && newObj._$protos[0] === 'HTMLDocument.prototype') {
145+
if (
146+
isNewObject &&
147+
(newObj._$protos[0] === 'HTMLDocument.prototype' ||
148+
newObj._$protos[0] === 'Document.prototype')
149+
) {
103150
newObj._$skipped = 'SKIPPED DOCUMENT';
104151
newObj._$type = 'HTMLDocument.prototype';
105152
return newObj;
@@ -234,6 +281,7 @@ function DomExtractor(selfName, pageMeta = {}) {
234281
if (obj === null || obj === undefined || !key) {
235282
return undefined;
236283
}
284+
debugPath(path);
237285
let accessException;
238286
const value = await new Promise(async (resolve, reject) => {
239287
let didResolve = false;
@@ -361,7 +409,15 @@ function DomExtractor(selfName, pageMeta = {}) {
361409
func = err.toString();
362410
}
363411
try {
364-
if (!doNotInvoke.includes(key) && !doNotInvoke.includes(path) && !value.prototype) {
412+
const isSvgPrototypeMethod =
413+
path.startsWith(`${selfName}.SVG`) && path.includes('.prototype.');
414+
if (
415+
!isSvgPrototypeMethod &&
416+
!doNotInvoke.includes(key) &&
417+
!doNotInvoke.includes(path) &&
418+
!value.prototype
419+
) {
420+
debugPath(`${path}()`);
365421
invocation = await new Promise(async (resolve, reject) => {
366422
const c = setTimeout(() => reject('Promise-like'), 650);
367423
let didReply = false;

0 commit comments

Comments
 (0)