From 492d71e865acd6bdc15a7b56052720fd9455ac79 Mon Sep 17 00:00:00 2001 From: Ben Myers Date: Tue, 17 Jun 2025 09:08:42 -0700 Subject: [PATCH 1/3] Extensive listing of android elements --- src/android.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/android.ts b/src/android.ts index 5290f3b..1cec0d3 100644 --- a/src/android.ts +++ b/src/android.ts @@ -17,6 +17,11 @@ interface UiAutomatorXmlNode { bounds?: string; hint?: string; focused?: string; + clickable?: string; + focusable?: string; + enabled?: string; + selected?: string; + package?: string; "content-desc"?: string; "resource-id"?: string; } @@ -207,9 +212,15 @@ export class AndroidRobot implements Robot { } } - if (node.text || node["content-desc"] || node.hint) { + // Include elements with text/labels OR clickable/focusable elements (like icons, buttons) + const hasTextOrLabel = node.text || node["content-desc"] || node.hint; + const isInteractive = node.clickable === "true" || node.focusable === "true" || + (node.class && (node.class.includes("Button") || node.class.includes("ImageView") || + node.class.includes("ImageButton") || node.class.includes("View"))); + + if (hasTextOrLabel || isInteractive) { const element: ScreenElement = { - type: node.class || "text", + type: node.class || "element", text: node.text, label: node["content-desc"] || node.hint || "", rect: this.getScreenElementRect(node), From b5f425892b6249ecd1f3b80104d9d4d5a895abdb Mon Sep 17 00:00:00 2001 From: Ben Myers Date: Tue, 17 Jun 2025 16:19:57 -0700 Subject: [PATCH 2/3] resource-id --- src/android.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android.ts b/src/android.ts index 1cec0d3..7e41f70 100644 --- a/src/android.ts +++ b/src/android.ts @@ -213,7 +213,7 @@ export class AndroidRobot implements Robot { } // Include elements with text/labels OR clickable/focusable elements (like icons, buttons) - const hasTextOrLabel = node.text || node["content-desc"] || node.hint; + const hasTextOrLabel = node.text || node["content-desc"] || node.hint || node["resource-id"]; const isInteractive = node.clickable === "true" || node.focusable === "true" || (node.class && (node.class.includes("Button") || node.class.includes("ImageView") || node.class.includes("ImageButton") || node.class.includes("View"))); From 59a76f36c6358066e59438de543036c6475cb0ef Mon Sep 17 00:00:00 2001 From: Ben Myers Date: Fri, 20 Jun 2025 16:12:14 -0700 Subject: [PATCH 3/3] feedback --- src/android.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android.ts b/src/android.ts index 7e41f70..c717803 100644 --- a/src/android.ts +++ b/src/android.ts @@ -222,7 +222,7 @@ export class AndroidRobot implements Robot { const element: ScreenElement = { type: node.class || "element", text: node.text, - label: node["content-desc"] || node.hint || "", + label: node["resource-id"] || node["content-desc"] || node.hint || "", rect: this.getScreenElementRect(node), };