Skip to content

Commit ae440a0

Browse files
Copilotsteveisok
authored and
matouskozak
committed
Android: Attempt emulator recovery before returning DEVICE_NOT_FOUND (dotnet#1550)
* Initial plan * Add emulator recovery to AdbRunner and update Android commands to attempt recovery before DEVICE_NOT_FOUND Co-authored-by: steveisok <471438+steveisok@users.noreply.github.com> * Android: Attempt emulator recovery before returning DEVICE_NOT_FOUND Co-authored-by: steveisok <471438+steveisok@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: steveisok <471438+steveisok@users.noreply.github.com>
1 parent f0c0b22 commit ae440a0

5 files changed

Lines changed: 313 additions & 11 deletions

File tree

‎src/Microsoft.DotNet.XHarness.Android/AdbRunner.cs‎

Lines changed: 220 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.IO;
99
using System.Linq;
1010
using System.Runtime.InteropServices;
11+
using System.Text;
1112
using System.Threading;
1213
using Microsoft.DotNet.XHarness.Android.Execution;
1314
using Microsoft.Extensions.Logging;
@@ -296,6 +297,220 @@ public void StartAdbServer()
296297

297298
public void KillAdbServer() => RunAdbCommand(new[] { "kill-server" }).ThrowIfFailed("Error killing ADB Server");
298299

300+
/// <summary>
301+
/// Attempts to recover the Android emulator when no suitable device is found.
302+
/// Steps:
303+
/// 1. Logs diagnostics about currently available devices
304+
/// 2. Resets the ADB daemon (kill-server + start-server)
305+
/// 3. If device still not visible, tries to restart the emulator process via
306+
/// systemctl (on Linux/systemd machines) or 'adb emu restart' as a fallback
307+
/// 4. Waits for the device to appear and complete boot
308+
/// </summary>
309+
/// <returns>true if a device appeared after recovery, false otherwise</returns>
310+
public bool TryRecoverEmulator()
311+
{
312+
// Step 1: Diagnostics - log what devices ARE currently attached
313+
_log.LogInformation("Attempting emulator recovery. Logging available devices for diagnostics...");
314+
var devicesResult = RunAdbCommand(new[] { "devices", "-l" }, TimeSpan.FromSeconds(30));
315+
_log.LogInformation($"Current 'adb devices -l' output:{Environment.NewLine}{devicesResult.StandardOutput}");
316+
317+
// On Linux, also log the emulator service status for diagnostics
318+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
319+
{
320+
try
321+
{
322+
var statusResult = RunSystemCommand("systemctl", new[] { "status", "android-emulator" }, TimeSpan.FromSeconds(15));
323+
_log.LogInformation($"systemctl status android-emulator:{Environment.NewLine}{statusResult.StandardOutput}");
324+
}
325+
catch (Exception e)
326+
{
327+
_log.LogDebug($"Unable to check systemctl status: {e.Message}");
328+
}
329+
}
330+
331+
// Step 2: Reset ADB daemon (kill-server + start-server)
332+
_log.LogInformation("Resetting ADB server as part of emulator recovery...");
333+
try
334+
{
335+
KillAdbServer();
336+
}
337+
catch (Exception e)
338+
{
339+
_log.LogWarning($"Error killing ADB server during recovery: {e.Message}");
340+
}
341+
342+
Thread.Sleep(TimeSpan.FromSeconds(2));
343+
344+
try
345+
{
346+
StartAdbServer();
347+
}
348+
catch (Exception e)
349+
{
350+
_log.LogWarning($"Error restarting ADB server during recovery: {e.Message}");
351+
}
352+
353+
// Step 3: Re-check for devices after ADB reset - if a device reappeared, we are done
354+
var recheckResult = RunAdbCommand(new[] { "devices", "-l" }, TimeSpan.FromSeconds(30));
355+
_log.LogInformation($"Devices after ADB server reset:{Environment.NewLine}{recheckResult.StandardOutput}");
356+
357+
var recheckLines = recheckResult.StandardOutput.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
358+
if (recheckLines.Length >= 2)
359+
{
360+
_log.LogInformation("Device(s) reappeared after ADB server reset; skipping emulator service restart");
361+
return true;
362+
}
363+
364+
// Step 4: Try to restart the emulator process
365+
bool restartAttempted = false;
366+
367+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
368+
{
369+
try
370+
{
371+
_log.LogInformation("Attempting to restart android-emulator service via systemctl...");
372+
var restartResult = RunSystemCommand("systemctl", new[] { "restart", "android-emulator" }, TimeSpan.FromMinutes(2));
373+
if (restartResult.Succeeded)
374+
{
375+
_log.LogInformation("systemctl restart android-emulator succeeded");
376+
restartAttempted = true;
377+
}
378+
else
379+
{
380+
_log.LogWarning($"systemctl restart android-emulator failed (exit code {restartResult.ExitCode}):{Environment.NewLine}{restartResult.StandardError}");
381+
}
382+
}
383+
catch (Exception e)
384+
{
385+
_log.LogDebug($"Unable to restart via systemctl: {e.Message}");
386+
}
387+
}
388+
389+
if (!restartAttempted)
390+
{
391+
// Fallback: attempt 'adb emu restart' (works if emulator console port is reachable)
392+
_log.LogInformation("Attempting 'adb emu restart' as emulator restart fallback...");
393+
try
394+
{
395+
var emuResult = RunAdbCommand(new[] { "emu", "restart" }, TimeSpan.FromSeconds(30));
396+
_log.LogDebug($"adb emu restart output: {emuResult.StandardOutput}");
397+
restartAttempted = true;
398+
}
399+
catch (Exception e)
400+
{
401+
_log.LogWarning($"adb emu restart failed: {e.Message}");
402+
}
403+
}
404+
405+
if (!restartAttempted)
406+
{
407+
_log.LogWarning("No emulator restart method succeeded; recovery not possible");
408+
return false;
409+
}
410+
411+
// Step 5: Wait for a device to reappear
412+
_log.LogInformation("Waiting for emulator to reappear after recovery (max 5 minutes)...");
413+
bool deviceAppeared = Retry(
414+
() =>
415+
{
416+
var r = RunAdbCommand(new[] { "devices", "-l" }, TimeSpan.FromSeconds(30));
417+
var lines = r.StandardOutput.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
418+
return lines.Length >= 2;
419+
},
420+
retryInterval: TimeSpan.FromSeconds(10),
421+
retryPeriod: TimeSpan.FromMinutes(5));
422+
423+
if (!deviceAppeared)
424+
{
425+
_log.LogWarning("No device appeared after emulator recovery attempt");
426+
return false;
427+
}
428+
429+
// Step 6: Wait for boot completion
430+
_log.LogInformation("Waiting for emulator boot completion after recovery...");
431+
var bootWatch = Stopwatch.StartNew();
432+
bool bootCompleted = Retry(
433+
() =>
434+
{
435+
string? bootStatus = GetDeviceProperty(AdbProperty.BootCompletion, null);
436+
_log.LogDebug($"sys.boot_completed = '{bootStatus}'");
437+
return bootStatus?.StartsWith('1') ?? false;
438+
},
439+
retryInterval: TimeSpan.FromSeconds(10),
440+
retryPeriod: TimeToWaitForBootCompletion);
441+
442+
if (bootCompleted)
443+
{
444+
_log.LogInformation($"Emulator recovered and boot completed after {(int)bootWatch.Elapsed.TotalSeconds} seconds");
445+
return true;
446+
}
447+
else
448+
{
449+
_log.LogWarning("Emulator appeared but did not complete boot within the timeout after recovery");
450+
return false;
451+
}
452+
}
453+
454+
/// <summary>
455+
/// Runs a non-ADB system command (e.g. systemctl) and returns the result.
456+
/// </summary>
457+
private static ProcessExecutionResults RunSystemCommand(string command, IEnumerable<string> arguments, TimeSpan timeout)
458+
{
459+
var processStartInfo = new ProcessStartInfo()
460+
{
461+
CreateNoWindow = true,
462+
UseShellExecute = false,
463+
RedirectStandardOutput = true,
464+
RedirectStandardError = true,
465+
FileName = command,
466+
};
467+
468+
foreach (var arg in arguments)
469+
{
470+
processStartInfo.ArgumentList.Add(arg);
471+
}
472+
473+
var p = new Process() { StartInfo = processStartInfo };
474+
var standardOut = new StringBuilder();
475+
var standardErr = new StringBuilder();
476+
477+
p.OutputDataReceived += (sender, e) => { if (e.Data != null) lock (standardOut) standardOut.AppendLine(e.Data); };
478+
p.ErrorDataReceived += (sender, e) => { if (e.Data != null) lock (standardErr) standardErr.AppendLine(e.Data); };
479+
480+
p.Start();
481+
p.BeginOutputReadLine();
482+
p.BeginErrorReadLine();
483+
484+
bool timedOut = false;
485+
int exitCode;
486+
487+
if (!p.WaitForExit((int)Math.Min(timeout.TotalMilliseconds, int.MaxValue)))
488+
{
489+
timedOut = true;
490+
exitCode = -1;
491+
try { p.Kill(); } catch { }
492+
}
493+
else
494+
{
495+
p.WaitForExit();
496+
exitCode = p.ExitCode;
497+
}
498+
499+
p.Close();
500+
501+
lock (standardOut)
502+
lock (standardErr)
503+
{
504+
return new ProcessExecutionResults()
505+
{
506+
ExitCode = exitCode,
507+
StandardOutput = standardOut.ToString(),
508+
StandardError = standardErr.ToString(),
509+
TimedOut = timedOut,
510+
};
511+
}
512+
}
513+
299514
public int CopyHeadlessFolder(string testPath, bool sharedRuntime = false)
300515
{
301516
_log.LogInformation($"Attempting to install {testPath}");
@@ -776,11 +991,15 @@ private IReadOnlyCollection<AndroidDevice> GetAllDevices(
776991

777992
if (requiredArchitectures?.Any() ?? false)
778993
{
994+
var availableDevices = devices;
779995
devices = devices.Where(device => device.SupportedArchitectures?.Intersect(requiredArchitectures).Any() ?? false).ToList();
780996

781997
if (devices.Count == 0)
782998
{
783-
_log.LogError($"No attached device supports one of required architectures {string.Join(", ", requiredArchitectures)}");
999+
var availableArchInfo = string.Join(", ", availableDevices.Select(
1000+
d => $"{d.DeviceSerial}=[{string.Join(",", d.SupportedArchitectures ?? Array.Empty<string>())}]"));
1001+
_log.LogError($"No attached device supports one of required architectures: {string.Join(", ", requiredArchitectures)}. " +
1002+
$"Found {availableDevices.Count} device(s) with: {availableArchInfo}");
7841003
return devices;
7851004
}
7861005
}

‎src/Microsoft.DotNet.XHarness.CLI/Commands/Android/AndroidInstallCommand.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,22 +91,36 @@ public static ExitCode InvokeHelper(
9191
// Make sure the adb server is started
9292
runner.StartAdbServer();
9393

94+
runner.TimeToWaitForBootCompletion = bootTimeoutSeconds;
95+
9496
AndroidDevice? device = runner.GetDevice(
9597
loadArchitecture: true,
9698
loadApiVersion: true,
9799
deviceId,
98100
apiVersion,
99101
requiredArchitectures);
100102

103+
if (device is null)
104+
{
105+
logger.LogWarning("No compatible device found on first attempt; trying emulator recovery...");
106+
if (runner.TryRecoverEmulator())
107+
{
108+
device = runner.GetDevice(
109+
loadArchitecture: true,
110+
loadApiVersion: true,
111+
deviceId,
112+
apiVersion,
113+
requiredArchitectures);
114+
}
115+
}
116+
101117
if (device is null)
102118
{
103119
throw new NoDeviceFoundException($"Failed to find compatible device: {string.Join(", ", requiredArchitectures)}");
104120
}
105121

106122
diagnosticsData.CaptureDeviceInfo(device);
107123

108-
runner.TimeToWaitForBootCompletion = bootTimeoutSeconds;
109-
110124
// Wait till at least device(s) are ready
111125
if (!runner.WaitForDevice())
112126
{

‎src/Microsoft.DotNet.XHarness.CLI/Commands/Android/AndroidRunCommand.cs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,30 @@ protected override ExitCode InvokeCommand(ILogger logger)
3939
// Make sure the adb server is started
4040
runner.StartAdbServer();
4141

42+
runner.TimeToWaitForBootCompletion = Arguments.LaunchTimeout;
43+
4244
var device = string.IsNullOrEmpty(Arguments.DeviceId.Value)
4345
? runner.GetSingleDevice(loadArchitecture: true, loadApiVersion: true, requiredInstalledApp: "package:" + Arguments.PackageName)
4446
: runner.GetSingleDevice(loadArchitecture: true, loadApiVersion: true, requiredDeviceId: Arguments.DeviceId.Value);
4547

48+
if (device is null)
49+
{
50+
logger.LogWarning("No compatible device found on first attempt; trying emulator recovery...");
51+
if (runner.TryRecoverEmulator())
52+
{
53+
device = string.IsNullOrEmpty(Arguments.DeviceId.Value)
54+
? runner.GetSingleDevice(loadArchitecture: true, loadApiVersion: true, requiredInstalledApp: "package:" + Arguments.PackageName)
55+
: runner.GetSingleDevice(loadArchitecture: true, loadApiVersion: true, requiredDeviceId: Arguments.DeviceId.Value);
56+
}
57+
}
58+
4659
if (device is null)
4760
{
4861
return ExitCode.DEVICE_NOT_FOUND;
4962
}
5063

5164
DiagnosticsData.CaptureDeviceInfo(device);
5265

53-
runner.TimeToWaitForBootCompletion = Arguments.LaunchTimeout;
54-
5566
// Wait till at least device(s) are ready
5667
if (!runner.WaitForDevice())
5768
{

‎src/Microsoft.DotNet.XHarness.CLI/Commands/AndroidHeadless/AndroidHeadlessInstallCommand.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,22 +86,36 @@ public static ExitCode InvokeHelper(
8686
// Make sure the adb server is started
8787
runner.StartAdbServer();
8888

89+
runner.TimeToWaitForBootCompletion = bootTimeoutSeconds;
90+
8991
AndroidDevice? device = runner.GetDevice(
9092
loadArchitecture: true,
9193
loadApiVersion: true,
9294
deviceId,
9395
apiVersion,
9496
testRequiredArchitecture);
9597

98+
if (device is null)
99+
{
100+
logger.LogWarning("No compatible device found on first attempt; trying emulator recovery...");
101+
if (runner.TryRecoverEmulator())
102+
{
103+
device = runner.GetDevice(
104+
loadArchitecture: true,
105+
loadApiVersion: true,
106+
deviceId,
107+
apiVersion,
108+
testRequiredArchitecture);
109+
}
110+
}
111+
96112
if (device is null)
97113
{
98114
throw new NoDeviceFoundException($"Failed to find compatible device: {string.Join(", ", testRequiredArchitecture)}");
99115
}
100116

101117
diagnosticsData.CaptureDeviceInfo(device);
102118

103-
runner.TimeToWaitForBootCompletion = bootTimeoutSeconds;
104-
105119
// Wait till at least device(s) are ready
106120
if (!runner.WaitForDevice())
107121
{

0 commit comments

Comments
 (0)