Selenium Download Files

Starting with Selenium version 4.8.0 you can now download files from grid nodes to your local environment for further validation. This new functionality is now built into Selenium and can be used as an alternative to our own file download solution.

Linux nodes require a custom download directory location, "/home/ubuntu/browser/downloads"

Windows nodes download all files by default to "C:\Users\Administrator\Downloads"

These download directories are emptied before each new test so the same node can run the same test again downloading the same file without duplicate/indexed file names.

NOTE: if you do download a file to local, typically keep the file size below 50 mb as base64 encoding do enlarge the file on the node when read and are subject to resource limitations.

NOTE: endpoints and functionality have changed starting with selenium version 4.8.1. See separate example if you are using version 4.8.0.

NOTE: This functionality has been changed to another implementation in selenium version 4.8.2+ and once its stable enough we will implement it in future versions.

Java Example

Selenium version 4.8.1

package download;

import static org.openqa.selenium.remote.http.Contents.string;

import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.commons.codec.binary.Base64;
import org.openqa.selenium.By;
import org.openqa.selenium.Platform;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.edge.EdgeOptions;
import org.openqa.selenium.firefox.FirefoxOptions;
import org.openqa.selenium.io.Zip;
import org.openqa.selenium.json.Json;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.remote.http.ClientConfig;
import org.openqa.selenium.remote.http.HttpClient;
import org.openqa.selenium.remote.http.HttpMethod;
import org.openqa.selenium.remote.http.HttpRequest;
import org.openqa.selenium.remote.http.HttpResponse;

import junit.framework.TestCase;

public class v4_download_files extends TestCase {

	private WebDriver driver;

	// NOTE: find these credentials in your Gridlastic dashboard after launching
	// your selenium grid (get a free account).
	// This is a Selenium 4 example
	
	// Linux nodes require a custom download directory location, /home/ubuntu/browser/downloads
	// Windows nodes download all files by default to C:\Users\Administrator\Downloads
	// These download directories are emptied before each new test so the same node
	// can run the same test again downloading the same file without duplicate/indexed file names.
	// NOTE: if you do download a file to local, typically keep the file size below 50 mb as base64 encoding do enlarge the file on the node when read and are subject to resource limitations.
	
	
	String video_url = System.getenv("VIDEO_URL");
	String hub = System.getenv("HUB"); // like "https://USERNAME:ACCESS_KEY@HUB_SUBDOMAIN.gridlastic.com/wd/hub";

	public void setUp() throws Exception {

		ChromeOptions options = new ChromeOptions();
		// FirefoxOptions options = new FirefoxOptions();
		// options.setCapability("browserVersion", "110"); // firefox
		options.setCapability("browserVersion", "latest");
		options.setCapability("platformName", Platform.WIN10);
		// options.setCapability("platformName", Platform.LINUX);

		// video recording of test
		Map gridlasticOptions = new HashMap<>();
		gridlasticOptions.put("video", true);
		options.setCapability("gridlastic:options", gridlasticOptions);

		// CHROME
		if (options.getBrowserName().toString().equalsIgnoreCase("chrome")) {
			
			if (options.getPlatformName().toString().equalsIgnoreCase("linux")) {
				Map prefs = new LinkedHashMap<>();
				prefs.put("download.default_directory", "/home/ubuntu/browser/downloads"); // custom download directory required.
				prefs.put("download.directory_upgrade", Boolean.valueOf(true));
				prefs.put("download.prompt_for_download", Boolean.valueOf(false));
				prefs.put("plugins.always_open_pdf_externally", Boolean.valueOf(true));
				prefs.put("safebrowsing_for_trusted_sources_enabled", Boolean.valueOf(false));
				options.setExperimentalOption("prefs", prefs);
			} else {
				// No need to set a custom download directory, all files downloaded to
				// C:\Users\Administrator\Downloads by default.				
				
				Map prefs = new LinkedHashMap<>();
				prefs.put("download.prompt_for_download", Boolean.valueOf(false));
				prefs.put("plugins.always_open_pdf_externally", Boolean.valueOf(true));
				prefs.put("safebrowsing_for_trusted_sources_enabled", Boolean.valueOf(false));
				options.setExperimentalOption("prefs", prefs);
			}
		}

		/*
		 * // FIREFOX
		 * if (options.getBrowserName().toString().equalsIgnoreCase("firefox")) { if
		 * (options.getPlatformName().toString().equalsIgnoreCase("linux")) {
		 * options.addPreference("browser.download.folderList", 2);
		 * options.addPreference("browser.download.dir",
		 * "/home/ubuntu/browser/downloads"); // custom download directory required.
		 * options.addPreference("browser.helperApps.alwaysAsk.force", false);
		 * options.addPreference("browser.download.manager.showWhenStarting", false);
		 * options.addPreference("browser.helperApps.neverAsk.saveToDisk",
		 * "application/download, application/octet-stream, text/csv, images/jpeg, application/pdf"
		 * ); options.addPreference("pdfjs.disabled", true); } else { // No need to set
		 * a custom download directory, all files downloaded to //
		 * C:\Users\Administrator\Downloads by default.
		 * options.addPreference("browser.download.folderList", 2);
		 * options.addPreference("browser.helperApps.alwaysAsk.force", false);
		 * options.addPreference("browser.download.manager.showWhenStarting", false);
		 * options.addPreference("browser.helperApps.neverAsk.saveToDisk",
		 * "application/download, application/octet-stream, text/csv, images/jpeg, application/pdf"
		 * ); options.addPreference("pdfjs.disabled", true); } }
		 */

		ClientConfig config = ClientConfig.defaultConfig().readTimeout(Duration.ofMinutes(10));
		driver = RemoteWebDriver.builder().address(new URL(hub)).oneOf(options).config(config).build();
	}

	public void test_download_file() throws Exception {

		driver.get("https://foundation.mozilla.org/en/who-we-are/public-records/");
		String fileName = "mf-articles-of-incorporation.pdf";
		WebElement element = driver.findElement(By.linkText("Articles of Incorporation of the Mozilla Foundation"));
		element.click();
		TimeUnit.SECONDS.sleep(30);

		URL gridUrl = new URL(hub);
		
		// Get downloaded file names
		HttpRequest request = new HttpRequest(HttpMethod.GET,
		String.format("/session/%s/se/files", ((RemoteWebDriver) driver).getSessionId()));		
		
		HttpClient client = HttpClient.Factory.createDefault().createClient(gridUrl);
		HttpResponse response = client.execute(request);
			
		Map raw = new Json().toType(string(response), Json.MAP_TYPE);
		System.out.println(raw.toString());
		Map map = Optional.ofNullable(raw.get("value")).map(data -> (Map) data)
				.orElseThrow(() -> new IllegalStateException("Could not find value attribute"));
		List files = (List) map.get("names");
	   
		if ((files).contains(fileName)) {
			System.out.println("Found downloaded file: " + fileName);
		}
		
	    
		// Download the file
		HttpRequest req = new HttpRequest(HttpMethod.POST, String.format("/session/%s/se/files", ((RemoteWebDriver) driver).getSessionId()));
		String payload = new Json().toJson(Collections.singletonMap("name", fileName));
		req.setContent(() -> new ByteArrayInputStream(payload.getBytes()));
		HttpResponse res = client.execute(req);
		
		Map raw_json = new Json().toType(string(res), Json.MAP_TYPE);
		Map map_value = Optional.ofNullable(
		        raw_json.get("value")
		      ).map(data -> (Map) data)
		      .orElseThrow(() -> new IllegalStateException("Could not find value attribute"));
		
		String encodedContents = map_value.get("contents").toString();
	    
	    
		// The file contents would always be a zip file and has to be unzipped.
		String home = System.getProperty("user.home");
		File dirToCopyTo = new File(home + "/downloads/gridnodes/" + ((RemoteWebDriver) driver).getSessionId() + "/");
		Zip.unzip(encodedContents, dirToCopyTo);
		client.close();

	}

	public void tearDown() throws Exception {
		System.out.println("GRIDLASTIC VIDEO URL: " + video_url + ((RemoteWebDriver) driver).getSessionId());
		driver.quit();
	}

}

Selenium version 4.8.0

package download;

import static org.openqa.selenium.remote.http.Contents.string;

import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.commons.codec.binary.Base64;
import org.openqa.selenium.By;
import org.openqa.selenium.Platform;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.firefox.FirefoxOptions;
import org.openqa.selenium.io.Zip;
import org.openqa.selenium.json.Json;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.remote.http.ClientConfig;
import org.openqa.selenium.remote.http.HttpClient;
import org.openqa.selenium.remote.http.HttpMethod;
import org.openqa.selenium.remote.http.HttpRequest;
import org.openqa.selenium.remote.http.HttpResponse;

import junit.framework.TestCase;

public class v4_download_files extends TestCase {

	private WebDriver driver;

	String video_url = System.getenv("VIDEO_URL");
	String hub = System.getenv("HUB"); // like "https://USERNAME:ACCESS_KEY@HUB_SUBDOMAIN.gridlastic.com/wd/hub";

	public void setUp() throws Exception {

		ChromeOptions options = new ChromeOptions();
		// FirefoxOptions options = new FirefoxOptions();
		// options.setCapability("browserVersion", "109"); // firefox
		options.setCapability("browserVersion", "latest");
		options.setCapability("platformName", Platform.WIN10);
		// options.setCapability("platformName", Platform.LINUX);

		// video recording of test
		Map gridlasticOptions = new HashMap<>();
		gridlasticOptions.put("video", true);
		options.setCapability("gridlastic:options", gridlasticOptions);

		// CHROME
		if (options.getBrowserName().toString().equalsIgnoreCase("chrome")) {
			
			if (options.getPlatformName().toString().equalsIgnoreCase("linux")) {
				Map prefs = new LinkedHashMap<>();
				prefs.put("download.default_directory", "/home/ubuntu/browser/downloads"); // custom download directory required.
				prefs.put("download.directory_upgrade", Boolean.valueOf(true));
				prefs.put("download.prompt_for_download", Boolean.valueOf(false));
				prefs.put("plugins.always_open_pdf_externally", Boolean.valueOf(true));
				prefs.put("safebrowsing_for_trusted_sources_enabled", Boolean.valueOf(false));
				options.setExperimentalOption("prefs", prefs);
			} else {
				// No need to set a custom download directory, all files downloaded to
				// C:\Users\Administrator\Downloads by default.				
				
				Map prefs = new LinkedHashMap<>();
				prefs.put("download.prompt_for_download", Boolean.valueOf(false));
				prefs.put("plugins.always_open_pdf_externally", Boolean.valueOf(true));
				prefs.put("safebrowsing_for_trusted_sources_enabled", Boolean.valueOf(false));
				options.setExperimentalOption("prefs", prefs);
			}
		}

		/*
		 * // FIREFOX
		 * if (options.getBrowserName().toString().equalsIgnoreCase("firefox")) { if
		 * (options.getPlatformName().toString().equalsIgnoreCase("linux")) {
		 * options.addPreference("browser.download.folderList", 2);
		 * options.addPreference("browser.download.dir",
		 * "/home/ubuntu/browser/downloads"); // custom download directory required.
		 * options.addPreference("browser.helperApps.alwaysAsk.force", false);
		 * options.addPreference("browser.download.manager.showWhenStarting", false);
		 * options.addPreference("browser.helperApps.neverAsk.saveToDisk",
		 * "application/download, application/octet-stream, text/csv, images/jpeg, application/pdf"
		 * ); options.addPreference("pdfjs.disabled", true); } else { // No need to set
		 * a custom download directory, all files downloaded to //
		 * C:\Users\Administrator\Downloads by default.
		 * options.addPreference("browser.download.folderList", 2);
		 * options.addPreference("browser.helperApps.alwaysAsk.force", false);
		 * options.addPreference("browser.download.manager.showWhenStarting", false);
		 * options.addPreference("browser.helperApps.neverAsk.saveToDisk",
		 * "application/download, application/octet-stream, text/csv, images/jpeg, application/pdf"
		 * ); options.addPreference("pdfjs.disabled", true); } }
		 */

		ClientConfig config = ClientConfig.defaultConfig().readTimeout(Duration.ofMinutes(10));
		driver = RemoteWebDriver.builder().address(new URL(hub)).oneOf(options).config(config).build();
	}

	public void test_download_file() throws Exception {

		driver.get("https://foundation.mozilla.org/en/who-we-are/public-records/");
		String fileName = "mf-articles-of-incorporation.pdf";
		WebElement element = driver.findElement(By.linkText("Articles of Incorporation of the Mozilla Foundation"));
		element.click();
		TimeUnit.SECONDS.sleep(30);

		HttpRequest request = new HttpRequest(HttpMethod.GET,
				String.format("/session/%s/se/file", ((RemoteWebDriver) driver).getSessionId()));
		request.addQueryParameter("filename", fileName);
		URL gridUrl = new URL(hub);
		HttpClient client = HttpClient.Factory.createDefault().createClient(gridUrl);
		HttpResponse response = client.execute(request);
		Map map = new Json().toType(string(response), Json.MAP_TYPE);
		String encodedContents = map.get("contents").toString();

		// The file contents would always be a zip file and has to be unzipped.
		String home = System.getProperty("user.home");
		File dirToCopyTo = new File(home + "/downloads/gridnodes/" + ((RemoteWebDriver) driver).getSessionId() + "/");
		Zip.unzip(encodedContents, dirToCopyTo);
		client.close();

	}

	public void tearDown() throws Exception {
		System.out.println("GRIDLASTIC VIDEO URL: " + video_url + ((RemoteWebDriver) driver).getSessionId());
		driver.quit();
	}

}