Skip to content

GitLab

A
appinventor-sources
Unverified Commit ae148869 authored by Evan W. Patton's avatar Evan W. Patton Committed by GitHub
Browse files
Options

Implement multidex in the buildserver (#2187) 

* Implement multidex in the buildserver

Change-Id: I1593438f80d59e9bcf6e7eb407ce902700bc67dc
parent 3757bc23
Hide whitespace changes
Inline Side-by-side
Showing with 234 additions and 125 deletions
appinventor/buildserver/build.xml
View file @ ae148869
......@@ -193,7 +193,7 @@
<sysproperty key="file.encoding" value="UTF-8" />
<jvmarg value="-Djava.awt.headless=true" />
<arg value="--childProcessRamMb" />
<arg value="1024" />
<arg value="2048" />
<arg value="--inputZipFile" />
<arg value="${local.build.dir}/aiplayapp.zip" />
<arg value="--userName" />
......@@ -226,7 +226,7 @@
<sysproperty key="file.encoding" value="UTF-8" />
<jvmarg value="-Djava.awt.headless=true" />
<arg value="--childProcessRamMb" />
<arg value="1024" />
<arg value="2048" />
<arg value="--inputZipFile" />
<arg value="${local.build.dir}/aiplayapp.zip" />
<arg value="--userName" />
......@@ -264,7 +264,7 @@
<sysproperty key="file.encoding" value="UTF-8" />
<jvmarg value="-Djava.awt.headless=true" />
<arg value="--childProcessRamMb" />
<arg value="1024" />
<arg value="2048" />
<arg value="--inputZipFile" />
<arg value="${local.build.dir}/aiplayapp.zip" />
<arg value="--userName" />
......
appinventor/buildserver/src/com/google/appinventor/buildserver/Compiler.java
View file @ ae148869
......@@ -6,6 +6,9 @@
package com.google.appinventor.buildserver;
import com.android.ide.common.internal.AaptCruncher;
import com.android.ide.common.internal.PngCruncher;
import com.android.sdklib.build.ApkBuilder;
import com.google.appinventor.buildserver.util.AARLibraries;
import com.google.appinventor.buildserver.util.AARLibrary;
import com.google.appinventor.components.common.ComponentDescriptorConstants;
......@@ -17,28 +20,21 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import com.android.ide.common.internal.AaptCruncher;
import com.android.ide.common.internal.PngCruncher;
import com.android.sdklib.build.ApkBuilder;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.codehaus.jettison.json.JSONTokener;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.geom.Ellipse2D;
import java.awt.geom.RoundRectangle2D;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
......@@ -53,15 +49,21 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.lang.Math;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.imageio.ImageIO;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.codehaus.jettison.json.JSONTokener;
/**
* Main entry point for the YAIL compiler.
......@@ -132,6 +134,14 @@ public final class Compiler {
private static final String APKSIGNER_JAR =
RUNTIME_FILES_DIR + "apksigner.jar";
private static final Set<String> CRITICAL_JARS =
new HashSet<>(Arrays.asList(
RUNTIME_FILES_DIR + "appcompat-v7.jar",
RUNTIME_FILES_DIR + "common.jar",
RUNTIME_FILES_DIR + "lifecycle-common.jar",
RUNTIME_FILES_DIR + "support-compat.jar"
));
private static final String LINUX_AAPT_TOOL =
"/tools/linux/aapt";
private static final String LINUX_ZIPALIGN_TOOL =
......@@ -274,13 +284,17 @@ public final class Compiler {
private File libsDir; // The directory that will contain any native libraries for packaging
private String dexCacheDir;
private boolean hasSecondDex = false; // True if classes2.dex should be added to the APK
private JSONArray simpleCompsBuildInfo;
private JSONArray extCompsBuildInfo;
private Set<String> simpleCompTypes; // types needed by the project
private Set<String> extCompTypes; // types needed by the project
/**
* A list of the dex files created by {@link #runMultidex}.
*/
private List<File> dexFiles = new ArrayList<>();
/**
* Mapping from type name to path in project to minimize tests against the file system.
*/
......@@ -861,20 +875,20 @@ public final class Compiler {
// to make the app available on devices that lack the feature. Without these lines the Play Store
// makes a guess based on permissions and assumes that they are required features.
if (isForCompanion) {
out.write(" <uses-feature android:name=\"android.hardware.bluetooth\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.telephony\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location.network\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location.gps\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.microphone\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.touchscreen\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.camera\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.camera.autofocus\" android:required=\"false\" />\n");
if (isForEmulator) {
out.write(" <uses-feature android:name=\"android.hardware.wifi\" android:required=\"false\" />\n"); // We actually require wifi
} else {
out.write(" <uses-feature android:name=\"android.hardware.wifi\" />\n"); // We actually require wifi
}
out.write(" <uses-feature android:name=\"android.hardware.bluetooth\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.telephony\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location.network\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.location.gps\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.microphone\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.touchscreen\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.camera\" android:required=\"false\" />\n");
out.write(" <uses-feature android:name=\"android.hardware.camera.autofocus\" android:required=\"false\" />\n");
if (isForEmulator) {
out.write(" <uses-feature android:name=\"android.hardware.wifi\" android:required=\"false\" />\n"); // We actually require wifi
} else {
out.write(" <uses-feature android:name=\"android.hardware.wifi\" />\n"); // We actually require wifi
}
}
int minSdk = Integer.parseInt((project.getMinSdk() == null) ? DEFAULT_MIN_SDK : project.getMinSdk());
......@@ -1308,7 +1322,7 @@ public final class Compiler {
// Android SDK's Dex Ant task
File tmpDir = createDir(buildDir, "tmp");
String dexedClassesDir = tmpDir.getAbsolutePath();
if (!compiler.runDx(classesDir, dexedClassesDir, false)) {
if (!compiler.runMultidex(classesDir, dexedClassesDir)) {
return false;
}
if (reporter != null) {
......@@ -1408,9 +1422,12 @@ public final class Compiler {
ApkBuilder apkBuilder =
new ApkBuilder(apkAbsolutePath, zipArchive,
dexedClassesDir + File.separator + "classes.dex", null, System.out);
if (hasSecondDex) {
apkBuilder.addFile(new File(dexedClassesDir + File.separator + "classes2.dex"),
"classes2.dex");
if (dexFiles.size() > 1) {
for (File f : dexFiles) {
if (!f.getName().equals("classes.dex")) {
apkBuilder.addFile(f, f.getName());
}
}
}
if (nativeLibsNeeded.size() != 0) { // Need to add native libraries...
apkBuilder.addNativeLibraries(libsDir);
......@@ -1843,113 +1860,184 @@ public final class Compiler {
return true;
}
private boolean runDx(File classesDir, String dexedClassesDir, boolean secondTry) {
List<File> libList = new ArrayList<File>();
List<File> inputList = new ArrayList<File>();
List<File> class2List = new ArrayList<File>();
inputList.add(classesDir); //this is a directory, and won't be cached into the dex cache
inputList.add(new File(getResource(SIMPLE_ANDROID_RUNTIME_JAR)));
inputList.add(new File(getResource(KAWA_RUNTIME)));
inputList.add(new File(getResource(ACRA_RUNTIME)));
for (String jar : SUPPORT_JARS) {
inputList.add(new File(getResource(jar)));
}
for (String lib : uniqueLibsNeeded) {
libList.add(new File(lib));
}
// BEGIN DEBUG -- XXX --
// System.err.println("runDx -- libraries");
// for (File aFile : inputList) {
// System.err.println(" inputList => " + aFile.getAbsolutePath());
// }
// for (File aFile : libList) {
// System.err.println(" libList => " + aFile.getAbsolutePath());
// }
// END DEBUG -- XXX --
// attach the jars of external comps to the libraries list
Set<String> addedExtJars = new HashSet<String>();
for (String type : extCompTypes) {
String sourcePath = getExtCompDirPath(type) + SIMPLE_ANDROID_RUNTIME_JAR;
if (!addedExtJars.contains(sourcePath)) {
libList.add(new File(sourcePath));
addedExtJars.add(sourcePath);
/**
* Processes recursively the directory pointed at by {@code dir} and adds any class files
* encountered to the {@code classes} set.
*
* @param dir the directory to examine for class files
* @param classes the Set used to record the classes
* @param root the root path where the recursion started, which gets stripped from the file name
* to determine the class name
*/
private void recordDirectoryForMainDex(File dir, Set<String> classes, String root) {
File[] files = dir.listFiles();
if (files == null) {
return;
}
for (File f : files) {
if (f.isDirectory()) {
recordDirectoryForMainDex(f, classes, root);
} else if (f.getName().endsWith(".class")) {
String className = f.getAbsolutePath().replace(root, "");
className = className.substring(0, className.length() - 6);
classes.add(className.replaceAll("/", "."));
}
}
}
int offset = libList.size();
// Note: The choice of 12 libraries is arbitrary. We note that things
// worked to put all libraries into the first classes.dex file when we
// had 16 libraries and broke at 17. So this is a conservative number
// to try.
if (!secondTry) { // First time through, try base + 12 libraries
if (offset > 12)
offset = 12;
} else {
offset = 0; // Add NO libraries the second time through!
/**
* Processes the JAR file pointed at by {@code file} and adds the contained class names to
* {@code classes}.
*
* @param file a File object pointing to a JAR file
* @param classes the Set used to record the classes
* @throws IOException if the input file cannot be read
*/
private void recordJarForMainDex(File file, Set<String> classes) throws IOException {
try (ZipInputStream is = new ZipInputStream(new FileInputStream(file))) {
ZipEntry entry;
while ((entry = is.getNextEntry()) != null) {
String className = entry.getName();
if (className.endsWith(".class")) {
className = className.substring(0, className.length() - 6);
classes.add(className.replaceAll("/", "."));
}
}
}
for (int i = 0; i < offset; i++) {
inputList.add(libList.get(i));
}
/**
* Examines the given file and records its classes for the main dex class list.
*
* @param file a File object pointing to a JAR file or a directory containing class files
* @param classes the Set used to record the classes
* @return the input file
* @throws IOException if the input file cannot be read
*/
private File recordForMainDex(File file, Set<String> classes) throws IOException {
if (file.isDirectory()) {
recordDirectoryForMainDex(file, classes, file.getAbsolutePath() + File.separator);
} else if (file.getName().endsWith(".jar")) {
recordJarForMainDex(file, classes);
}
return file;
}
if (libList.size() - offset > 0) { // Any left over for classes2?
for (int i = offset; i < libList.size(); i++) {
class2List.add(libList.get(i));
/**
* Writes out the class list for the main dex file. The format of this file is the pathname of
* the class, including the .class extension, one per line.
*
* @param classesDir directory to place the main classes list
* @param classes the set of classes to include in the main dex file
* @return the path to the file containing the main classes list
*/
private String writeClassList(File classesDir, Set<String> classes) {
File target = new File(classesDir, "main-classes.txt");
try (PrintStream out = new PrintStream(new FileOutputStream(target))) {
for (String name : new TreeSet<>(classes)) {
out.println(name.replaceAll("\\.", "/") + ".class");
}
return target.getAbsolutePath();
} catch (IOException e) {
return null;
}
}
DexExecTask dexTask = new DexExecTask();
dexTask.setExecutable(getResource(DX_JAR));
dexTask.setOutput(dexedClassesDir + File.separator + "classes.dex");
dexTask.setChildProcessRamMb(childProcessRamMb);
if (dexCacheDir == null) {
dexTask.setDisableDexMerger(true);
} else {
createDir(new File(dexCacheDir));
dexTask.setDexedLibs(dexCacheDir);
}
/**
* Compiles Java class files and JAR files into the Dex file format using dx.
*
* @param classesDir directory containing compiled App Inventor screens
* @param dexedClassesDir output directory for classes.dex
* @return true if successful or false if an error occurred
*/
private boolean runMultidex(File classesDir, String dexedClassesDir) {
Set<String> mainDexClasses = new HashSet<>();
List<File> inputList = new ArrayList<>();
boolean success;
try {
// Set up classes for main dex file
inputList.add(recordForMainDex(classesDir, mainDexClasses));
inputList.add(recordForMainDex(new File(getResource(SIMPLE_ANDROID_RUNTIME_JAR)),
mainDexClasses));
inputList.add(recordForMainDex(new File(getResource(KAWA_RUNTIME)), mainDexClasses));
for (String jar : CRITICAL_JARS) {
inputList.add(recordForMainDex(new File(getResource(jar)), mainDexClasses));
}
long startDx = System.currentTimeMillis();
// Using System.err and System.out on purpose. Don't want to pollute build messages with
// tools output
boolean dxSuccess;
synchronized (SYNC_KAWA_OR_DX) {
setProgress(50);
dxSuccess = dexTask.execute(inputList);
if (dxSuccess && (class2List.size() > 0)) {
setProgress(60);
dexTask.setOutput(dexedClassesDir + File.separator + "classes2.dex");
inputList = new ArrayList<File>();
dxSuccess = dexTask.execute(class2List);
// Only include ACRA for the companion app
if (isForCompanion) {
inputList.add(recordForMainDex(new File(getResource(ACRA_RUNTIME)), mainDexClasses));
}
for (String jar : SUPPORT_JARS) {
if (CRITICAL_JARS.contains(jar)) { // already covered above
continue;
}
inputList.add(new File(getResource(jar)));
}
// Add the rest of the libraries in any order
for (String lib : uniqueLibsNeeded) {
inputList.add(new File(lib));
}
// Add extension libraries
Set<String> addedExtJars = new HashSet<>();
for (String type : extCompTypes) {
String sourcePath = getExtCompDirPath(type) + SIMPLE_ANDROID_RUNTIME_JAR;
if (!addedExtJars.contains(sourcePath)) {
inputList.add(new File(sourcePath));
addedExtJars.add(sourcePath);
}
}
// Run the dx utility
DexExecTask dexTask = new DexExecTask();
dexTask.setExecutable(getResource(DX_JAR));
dexTask.setMainDexClassesFile(writeClassList(classesDir, mainDexClasses));
dexTask.setOutput(dexedClassesDir);
dexTask.setChildProcessRamMb(childProcessRamMb);
if (dexCacheDir == null) {
dexTask.setDisableDexMerger(true);
} else {
createDir(new File(dexCacheDir));
dexTask.setDexedLibs(dexCacheDir);
}
String dxTimeMessage;
synchronized (SYNC_KAWA_OR_DX) {
setProgress(50);
long startDx = System.currentTimeMillis();
success = dexTask.execute(inputList);
dxTimeMessage = String.format(Locale.getDefault(), "DX time: %f seconds",
(System.currentTimeMillis() - startDx) / 1000.0);
setProgress(75);
hasSecondDex = true;
} else if (!dxSuccess) { // The initial dx blew out, try more conservative
LOG.info("DX execution failed, trying with fewer libraries.");
if (secondTry) { // Already tried the more conservative approach!
LOG.warning("YAIL compiler - DX execution failed (secondTry!).");
err.println("YAIL compiler - DX execution failed.");
userErrors.print(String.format(ERROR_IN_STAGE, "DX"));
return false;
} else {
return runDx(classesDir, dexedClassesDir, true);
}
// Aggregate all of the classes.dex files output by dx
File[] files = new File(dexedClassesDir).listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".dex");
}
});
if (files == null) {
throw new FileNotFoundException("Could not find classes.dex");
}
Collections.addAll(dexFiles, files);
// Log status
out.println(dxTimeMessage);
LOG.info(dxTimeMessage);
} catch (IOException e) {
// Error will be reported below
success = false;
}
if (!dxSuccess) {
if (!success) {
LOG.warning("YAIL compiler - DX execution failed.");
err.println("YAIL compiler - DX execution failed.");
userErrors.print(String.format(ERROR_IN_STAGE, "DX"));
return false;
}
String dxTimeMessage = "DX time: " +
((System.currentTimeMillis() - startDx) / 1000.0) + " seconds";
out.println(dxTimeMessage);
LOG.info(dxTimeMessage);
return true;
return success;
}
private boolean runAaptPackage(File manifestFile, File resDir, String tmpPackageName, File sourceOutputDir, File symbolOutputDir) {
......
appinventor/buildserver/src/com/google/appinventor/buildserver/DexExecTask.java
View file @ ae148869
......@@ -44,6 +44,8 @@ public class DexExecTask {
private int mChildProcessRamMb = 1024;
private boolean mDisableDexMerger = false;
private static Map<String, String> alreadyChecked = new HashMap<String, String>();
private String mainDexFile = null;
private boolean mPredex = true;
private static final Object semaphore = new Object(); // Used to protect dex cache creation
......@@ -66,6 +68,13 @@ public class DexExecTask {
mVerbose = verbose;
}
public void setMainDexClassesFile(String classList) {
mainDexFile = classList;
if (classList != null) {
mPredex = false;
}
}
/**
* Sets the value of the "output" attribute.
*
......@@ -79,6 +88,10 @@ public class DexExecTask {
mDexedLibs = dexedLibs;
}
public void setPredex(boolean predex) {
mPredex = predex;
}
/**
* Sets the value of the "nolocals" attribute.
*
......@@ -164,8 +177,10 @@ public class DexExecTask {
public boolean execute(List<File> paths) {
// pre dex libraries if needed
boolean successPredex = preDexLibraries(paths);
if (!successPredex) return false;
if (mPredex) {
boolean successPredex = preDexLibraries(paths);
if (!successPredex) return false;
}
System.out.println(String.format(
"Converting compiled files and external libraries into %1$s...", mOutput));
......@@ -189,6 +204,12 @@ public class DexExecTask {
commandLineList.add("--dex");
commandLineList.add("--positions=lines");
if (mainDexFile != null) {
commandLineList.add("--multi-dex");
commandLineList.add("--main-dex-list=" + mainDexFile);
commandLineList.add("--minimal-main-dex");
}
if (mNoLocals) {
commandLineList.add("--no-locals");
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment