feat(computer): add rdp demo (#95)

Author: quanru

.gitignore

@@ -16,6 +16,7 @@ test-results/
 .env
 
 output/
+docs/
 
 .DS_Store
 

README.md

@@ -18,6 +18,13 @@ Here are some examples you can refer to:
 - [Vitest Demo](./android/vitest-demo/): Integrate Midscene with Android and Vitest.
 - [YAML Scripts Demo](./android/yaml-scripts-demo/): Automate Android with scripts in YAML. This is the easiest way to integrate Midscene with your existing Android project.
 
+### Computer
+- [JavaScript SDK Demo](./computer/javascript-sdk-demo/): Integrate Midscene with your local desktop through `@midscene/computer`.
+- [Vitest Demo](./computer/vitest-demo/): Integrate Midscene with `@midscene/computer` and Vitest.
+- [Electron Demo](./computer/electron-demo/): Use Midscene with an Electron app.
+- [YAML Scripts Demo](./computer/yaml-scripts-demo/): Automate desktop tasks with scripts in YAML.
+- [RDP Demo](./computer/rdp-demo/): Control a remote Windows desktop directly over the RDP protocol.
+
 ## Connectivity Test
 
 - [Connectivity Test](./connectivity-test/): Use this folder to test the connectivity of the LLM Service.

computer/rdp-demo/README.md

@@ -0,0 +1,55 @@
+# Computer RDP Demo
+
+This demo shows how to use `@midscene/computer` to control a remote Windows desktop directly over the RDP protocol.
+
+## Steps
+
+### Preparation
+
+Create `.env` file with the same Midscene model variables used by other examples:
+
+```shell
+MIDSCENE_MODEL_BASE_URL="https://.../compatible-mode/v1"
+MIDSCENE_MODEL_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
+MIDSCENE_MODEL_NAME="qwen3-vl-plus"
+MIDSCENE_MODEL_FAMILY="qwen3-vl"
+```
+
+Then open [demo.ts](./demo.ts) and update the placeholder values in the `rdpTarget` object:
+
+```ts
+const rdpTarget = {
+  host: 'REPLACE_WITH_YOUR_RDP_HOST',
+  port: 3389,
+  username: 'Admin',
+  password: 'REPLACE_WITH_YOUR_RDP_PASSWORD',
+  ignoreCertificate: true,
+  adminSession: false,
+  domain: undefined,
+  securityProtocol: 'auto',
+};
+```
+
+The demo validates the RDP target values before creating the agent. If you leave a placeholder unchanged, it will throw immediately.
+
+If you want to use another model, refer to:
+https://midscenejs.com/model-common-config.html
+
+### Run demo
+
+```bash
+npm install
+
+npm run test
+```
+
+The demo will connect to the remote Windows desktop, wait for the remote framebuffer to become visible if the first frame is blank, open the Settings app, continue into the Windows Update page, read the visible page title and status summary back into structured JSON, and print the generated report path.
+
+## Notes
+
+- This demo uses protocol-level RDP control. Midscene sees the remote Windows framebuffer directly instead of controlling a local RDP client window.
+
+## Reference
+
+- https://midscenejs.com/computer-introduction.html
+- https://midscenejs.com/computer-api-reference.html

computer/rdp-demo/demo.ts

@@ -0,0 +1,103 @@
+import {
+  agentForRDPComputer,
+  type RDPComputerAgentOpt,
+} from '@midscene/computer';
+import 'dotenv/config';
+
+const rdpTarget = {
+  host: 'REPLACE_WITH_YOUR_RDP_HOST',
+  port: 3389,
+  username: 'Admin',
+  password: 'REPLACE_WITH_YOUR_RDP_PASSWORD',
+  ignoreCertificate: true,
+  adminSession: false,
+  domain: undefined,
+  securityProtocol: 'auto',
+} satisfies RDPComputerAgentOpt;
+
+function validateDemoConfig() {
+  for (const [key, value] of Object.entries({
+    host: rdpTarget.host,
+    password: rdpTarget.password,
+  })) {
+    const trimmed = value?.trim();
+    if (!trimmed || trimmed.startsWith('REPLACE_WITH_')) {
+      throw new Error(
+        `Please update rdpTarget.${key} in computer/rdp-demo/demo.ts before running the demo.`,
+      );
+    }
+  }
+}
+
+async function waitForRemoteDesktopReady(
+  agent: Awaited<ReturnType<typeof agentForRDPComputer>>,
+  maxAttempts = 10,
+  delayMs = 3_000,
+) {
+  let lastError: unknown;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      await agent.aiAssert(
+        'The remote screenshot is not a blank white screen. Some visible Windows UI such as the taskbar, desktop icons, the Start menu, or an application window is present.',
+      );
+      return;
+    } catch (error) {
+      lastError = error;
+      if (attempt === maxAttempts) {
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+  }
+
+  throw new Error(
+    `The remote desktop never became visible after ${maxAttempts} attempts.`,
+    { cause: lastError },
+  );
+}
+
+async function main() {
+  validateDemoConfig();
+
+  const agent = await agentForRDPComputer({
+    ...rdpTarget,
+    aiActionContext:
+      'You are controlling a remote Windows desktop directly through the RDP protocol. Every screenshot and action comes from the remote machine itself.',
+    generateReport: true,
+  });
+
+  try {
+    await waitForRemoteDesktopReady(agent);
+
+    await agent.aiAct(
+      'Click the Windows Start button, open the Settings app, then navigate to the Windows Update page. Stop only after the Windows Update page is clearly visible in the remote screenshot.',
+    );
+
+    await agent.aiAssert(
+      'The Windows Update page inside the Settings app is open and visible in the remote screenshot.',
+    );
+
+    const windowsUpdateSummary = await agent.aiQuery<{
+      pageTitle: string;
+      statusSummary: string;
+    }>(
+      '{pageTitle: string, statusSummary: string}, read the visible Windows Update page and return its main page title and the short status summary shown near the top of the page.',
+    );
+
+    console.log('Connected to remote desktop:', rdpTarget.host);
+    console.log('Windows Update page title:', windowsUpdateSummary.pageTitle);
+    console.log('Windows Update status:', windowsUpdateSummary.statusSummary);
+  } finally {
+    await agent.destroy();
+  }
+
+  if (agent.reportFile) {
+    console.log('Report saved to:', agent.reportFile);
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

computer/rdp-demo/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "computer-rdp-demo",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Demo for controlling a remote Windows desktop over RDP with Midscene",
+  "type": "module",
+  "scripts": {
+    "test": "tsx demo.ts"
+  },
+  "author": "",
+  "license": "MIT",
+  "devDependencies": {
+    "@midscene/computer": "latest",
+    "dotenv": "^16.4.5",
+    "tsx": "4.20.1"
+  }
+}