Fix XMLTextDecode issues with CDATA section (#1807)

* Migrate Android JSON lib tests to use Robolectric and Junit4

Change-Id: Icf472c1b694cd7b19436b7b55626e8bc0eba9f0b

* Repackage org.json dependency to prevent link issue on Android

Change-Id: I1913914f3182f14963d7531115a30af08e4d9624

appinventor/components/build.xml

@@ -146,7 +146,7 @@
     <copy toFile="${public.deps.dir}/google-http-client-android2-beta.jar" file="${lib.dir}/oauth/google-http-client-android2-1.10.3-beta.jar" />
     <copy toFile="${public.deps.dir}/google-http-client-android3-beta.jar" file="${lib.dir}/oauth/google-http-client-android3-1.10.3-beta.jar" />
     <copy toFile="${public.deps.dir}/gson-2.1.jar" file="${lib.dir}/gson/gson-2.1.jar" />
-    <copy toFile="${public.deps.dir}/json.jar" file="${lib.dir}/json/json.jar" />
+    <copy toFile="${public.deps.dir}/json.jar" file="${lib.dir}/json/json-android.jar" />
     <copy toFile="${public.deps.dir}/google-oauth-client-beta.jar" file="${lib.dir}/oauth/google-oauth-client-1.10.1-beta.jar" />
     <copy toFile="${public.deps.dir}/jedis.jar" file="${lib.dir}/jedis/jedis-3.0.0-SNAPSHOT-jar-with-dependencies.jar" />
     <copy toFile="${public.deps.dir}/commons-pool.jar" file="${lib.dir}/commons-pool/commons-pool2-2.0.jar" />

appinventor/components/src/com/google/appinventor/components/runtime/Web.java

@@ -37,7 +37,7 @@ import android.util.Log;
 
 import org.json.JSONException;
 import org.json.JSONObject;
-import org.json.XML;
+import com.google.appinventor.components.runtime.repackaged.org.json.XML;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -807,9 +807,8 @@ public class Web extends AndroidNonvisibleComponent implements Component {
   // HTML for the component documentation on the Web.  It's too long for a tooltip, anyway.
   public Object XMLTextDecode(String XmlText) {
     try {
-      JSONObject json = XML.toJSONObject(XmlText);
-      return JsonTextDecode(json.toString());
-    } catch (JSONException e) {
+      return JsonTextDecode(XML.toJSONObject(XmlText).toString());
+    } catch (com.google.appinventor.components.runtime.repackaged.org.json.JSONException e) {
       // We could be more precise and signal different errors for the conversion to JSON
       // versus the decoding of that JSON, but showing the actual error message should
       // be good enough.

appinventor/components/src/com/google/appinventor/components/scripts/ExternalComponentGenerator.java

@@ -165,13 +165,21 @@ public class ExternalComponentGenerator {
     JSONArray buildInfos = new JSONArray();
     for (ExternalComponentInfo info : extensions) {
       JSONObject componentBuildInfo = info.buildInfo;
-      JSONArray librariesNeeded = componentBuildInfo.getJSONArray("libraries");
-      for (int j = 0; j < librariesNeeded.length(); ++j) { // Copy Library files for Unjar and Jaring
-        String library = librariesNeeded.getString(j);
-        copyFile(buildServerClassDirPath + File.separator + library,
-            extensionTempDirPath + File.separator + library);
+      try {
+        JSONArray librariesNeeded = componentBuildInfo.getJSONArray("libraries");
+        for (int j = 0; j < librariesNeeded.length(); ++j) {
+          // Copy Library files for Unjar and Jaring
+          String library = librariesNeeded.getString(j);
+          copyFile(buildServerClassDirPath + File.separator + library,
+              extensionTempDirPath + File.separator + library);
+        }
+        //empty the libraries meta-data to avoid redundancy
+        componentBuildInfo.put("libraries", new JSONArray());
+      } catch(JSONException e) {
+        // bad
+        throw new IllegalStateException("Unexpected JSON exception parsing simple_components.json",
+            e);
       }
-      componentBuildInfo.put("libraries", new JSONArray()); //empty the libraries meta-data to avoid redundancy
       buildInfos.put(componentBuildInfo);
     }
 
@@ -197,7 +205,7 @@ public class ExternalComponentGenerator {
     try {
       extensionBuildInfoFile = new FileWriter(extensionFileDirPath + File.separator + "component_build_info.json");
       extensionBuildInfoFile.write(buildInfos.get(0).toString());
-    } catch (IOException e) {
+    } catch (IOException|JSONException e) {
       e.printStackTrace();
     } finally {
       if (extensionBuildInfoFile != null) {

appinventor/components/tests/com/google/appinventor/components/runtime/WebTest.java

@@ -9,25 +9,36 @@ package com.google.appinventor.components.runtime;
 import com.google.appinventor.components.runtime.util.ErrorMessages;
 import com.google.appinventor.components.runtime.util.YailList;
 
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 
 /**
  * Tests Web.java.
  *
  * @author lizlooney@google.com (Liz Looney)
  */
-public class WebTest extends TestCase {
+@RunWith(RobolectricTestRunner.class)
+@Config(sdk = 23, manifest="tests/AndroidManifest.xml")
+public class WebTest {
   private Web web;
 
-  @Override
-  protected void setUp() throws Exception {
+  @Before
+  public void setUp() throws Exception {
     web = new Web();
   }
 
+  @Test
   public void testDecodeJsonText() throws Exception {
     // String values.
     assertEquals("\t tab \t tab \t",
@@ -42,8 +53,6 @@ public class WebTest extends TestCase {
         web.decodeJsonText("\"\\\" quote \\\" quote \\\"\""));
     assertEquals("~ encoded tilda ~ encoded tilda ~",
         web.decodeJsonText("\"\\u007E encoded tilda \\u007E encoded tilda \\u007E\""));
-    assertEquals("A normal string without quotes.",
-        web.decodeJsonText("A normal string without quotes."));
 
     // Boolean values.
     assertEquals(Boolean.TRUE, web.decodeJsonText("True"));
@@ -90,6 +99,7 @@ public class WebTest extends TestCase {
     }
   }
 
+  @Test
   public void testDecodeXMLText() throws Exception {
     Object decodedObject = web.XMLTextDecode("<foo>123</foo>");
     // should be the list of one element, which is a pair of "foo" and 123
@@ -105,6 +115,7 @@ public class WebTest extends TestCase {
     assertEquals(123, pair.get(1));
   }
 
+  @Test
   public void testDecodeXMLText2() throws Exception {
     Object decodedObject = web.XMLTextDecode("<a><foo>1 2 3</foo><bar>456</bar></a>");
     // should be the list of one element, which is a pair of "a" and a list X.
@@ -136,6 +147,22 @@ public class WebTest extends TestCase {
     assertEquals("1 2 3", secondPair.get(1));
   }
 
+  @Test
+  public void testDecodeXMLCDATA() throws Exception {
+    Object decodedObject = web.XMLTextDecode("<xml><![CDATA[foo < bar || bar > baz]]></xml>");
+    assertTrue(decodedObject instanceof List);
+    List outerList = (List) decodedObject;
+    assertEquals(1, outerList.size());
+    assertTrue(outerList.get(0) instanceof List);
+    List tagValuePair = (List) ((List) decodedObject).get(0);
+    assertEquals(2, tagValuePair.size());
+    // tag should be xml
+    assertEquals("xml", tagValuePair.get(0));
+    // value should be decoded CDATA
+    assertEquals("foo < bar || bar > baz", tagValuePair.get(1));
+  }
+
+  @Test
   public void testbuildRequestData() throws Exception {
     List<Object> list = new ArrayList<Object>();
     list.add(YailList.makeList(new String[] { "First Name", "Barack" }));

appinventor/components/tests/com/google/appinventor/components/runtime/util/GeoJSONUtilTest.java

@@ -19,6 +19,7 @@ import com.google.appinventor.components.runtime.util.MapFactory.MapFeature;
 import com.google.appinventor.components.runtime.util.MapFactory.MapMarker;
 import gnu.lists.FString;
 import gnu.lists.LList;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.Test;
 import org.osmdroid.util.GeoPoint;
@@ -294,7 +295,7 @@ public class GeoJSONUtilTest extends MapTestBase {
   }
 
   @Test
-  public void testWriteFeaturesAsGeoJSON() throws IOException {
+  public void testWriteFeaturesAsGeoJSON() throws IOException, JSONException {
     MarkerTest.createMarker(getMap(), 42.0, -71.0);
     defaultLineEW(new LineString(getMap()));
     defaultPolygon(new Polygon(getMap()));
@@ -308,7 +309,7 @@ public class GeoJSONUtilTest extends MapTestBase {
   }
 
   @Test
-  public void testWriteFeaturesAsGeoJSONNoFeatures() throws IOException {
+  public void testWriteFeaturesAsGeoJSONNoFeatures() throws IOException, JSONException {
     String contents = saveMapToString();
     JSONObject json = new JSONObject(contents);
     assertEquals(0, json.getJSONArray("features").length());

appinventor/components/tests/com/google/appinventor/components/runtime/util/JsonUtilTest.java

@@ -6,26 +6,33 @@
 
 package com.google.appinventor.components.runtime.util;
 
-import junit.framework.TestCase;
-
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
+
 /**
  * Tests JsonUtil class.
  *
  *
  */
-public class JsonUtilTest extends TestCase {
+@RunWith(RobolectricTestRunner.class)
+@Config(sdk = 23, manifest="tests/AndroidManifest.xml")
+public class JsonUtilTest {
 
   // The elements in this next test were commented out due to a change in the
   // Json.javq library that makes geStringListFromJsonArray throw an error
   // if the item being operated on is not a string.
+  @Test
   public void testGetStringListFromJsonArray() throws JSONException {
 //   Object[] firstArray = {"Houston", "we", "have", "a", "problem"};
 //   Object[] secondArray = {"China", "we", "have", "an", "ultimatum"};
@@ -33,7 +40,7 @@ public class JsonUtilTest extends TestCase {
     List<String> mixedList = new ArrayList<String>();
     mixedList.add("Hello.");
     mixedList.add("O hi.");
-    mixedList.add(new Integer(9).toString());
+    mixedList.add(Integer.toString(9));
 //    mixedList.add(new JSONArray(Arrays.asList(firstArray)).toString());
 //    mixedList.add(new JSONArray(Arrays.asList(secondArray)).toString());
 
@@ -41,7 +48,7 @@ public class JsonUtilTest extends TestCase {
       "[" +
       "\"Hello.\"," +
       "\"O hi.\"," +
-      "\"9\","  +
+      "\"9\""  +
       //    "[\"Houston\",\"we\",\"have\",\"a\",\"problem\"]," +
       //   "[\"China\",\"we\",\"have\",\"an\",\"ultimatum\"]" +
       "]";
@@ -52,6 +59,7 @@ public class JsonUtilTest extends TestCase {
     }
   }
 
+  @Test
   public void testGetListFromJsonArray() throws JSONException {
     Object[] firstArray = {"Houston", "we", "have", "a", "problem"};
     List<Object> firstList = Arrays.asList(firstArray);
@@ -76,7 +84,7 @@ public class JsonUtilTest extends TestCase {
       "[\"China\",\"we\",\"have\",\"an\",\"ultimatum\"]," +
       "9.5," +
       "true," +
-      "\"faLse\"," +
+      "\"faLse\"" +
       "]";
     List<Object> testList;
     testList = JsonUtil.getListFromJsonArray(new JSONArray(jsonInput));
@@ -85,6 +93,7 @@ public class JsonUtilTest extends TestCase {
     }
   }
 
+  @Test
   public void testGetListFromJsonObject() throws JSONException {
     String jsonInput = "{\"a\": 1, \"c\": [\"a\", \"b\", \"c\"], " +
         "\"b\": \"boo\", \"d\": {\"e\": \"f\"}}";
@@ -104,6 +113,7 @@ public class JsonUtilTest extends TestCase {
     assertEquals(returnList.get(3), dList);
   }
 
+  @Test
   public void testConvertBoolean() throws JSONException {
     assertEquals(true, JsonUtil.convertJsonItem("true"));
     assertEquals(true, JsonUtil.convertJsonItem(true));
@@ -113,6 +123,7 @@ public class JsonUtilTest extends TestCase {
     assertEquals(false, JsonUtil.convertJsonItem("faLse"));
   }
 
+  @Test
   public void testConvertNumber() throws JSONException {
     String jsonInput = "[9.5,-9.5,9,-9,123456789101112,0xF]";
     JSONArray array = new JSONArray(jsonInput);
@@ -121,11 +132,10 @@ public class JsonUtilTest extends TestCase {
     assertEquals(9, JsonUtil.convertJsonItem(array.get(2)));
     assertEquals(-9, JsonUtil.convertJsonItem(array.get(3)));
     assertEquals(123456789101112L, JsonUtil.convertJsonItem(array.get(4)));
-//    assertEquals(15, JsonUtil.convertJsonItem(array.get(5)));
-//    The above line used to work before the JSON library was changed.
-    assertEquals("0xF", JsonUtil.convertJsonItem(array.get(5)));
+    assertEquals(15, JsonUtil.convertJsonItem(array.get(5)));
   }
-  
+
+  @Test
   public void testConvertEmpty() throws JSONException {
     Object shouldBeEmpty = JsonUtil.getObjectFromJson("");
     assertEquals("", JsonUtil.getObjectFromJson(""));

appinventor/components/tests/com/google/appinventor/components/runtime/util/YailListTest.java

@@ -9,17 +9,27 @@ package com.google.appinventor.components.runtime.util;
 import gnu.lists.FString;
 import gnu.math.IntNum;
 
-import junit.framework.TestCase;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
 
 import java.util.ArrayList;
 import java.util.HashSet;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 /**
  * Tests YailList class.
  *
  */
-public class YailListTest extends TestCase {
+@RunWith(RobolectricTestRunner.class)
+@Config(sdk = 23, manifest="tests/AndroidManifest.xml")
+public class YailListTest {
 
+  @Test
   public void testEmptyList() {
     YailList yailList = new YailList();
     assertEquals(0, yailList.size());
@@ -39,6 +49,7 @@ public class YailListTest extends TestCase {
     }
   }
 
+  @Test
   public void testToString() {
     Object[] object = {"Houston", "we", "have", "a", "problem"};
     YailList yailList = YailList.makeList(object);
@@ -55,6 +66,7 @@ public class YailListTest extends TestCase {
     assertEquals("(4 [one, two, three] 6)", yailList.toString());
   }
 
+  @Test
   public void testToStringArray() {
     Object[] object = {"Houston", "we", "have", "a", "problem"};
     YailList yailList = YailList.makeList(object);
@@ -64,17 +76,20 @@ public class YailListTest extends TestCase {
     }
   }
 
+  @Test
   public void testEmptyJsonStringOutput() {
     YailList yailList = new YailList();
     assertEquals("[]", yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutput() {
     Object[] object = {"Houston", "we", "have", "a", "problem"};
     YailList yailList = YailList.makeList(object);
     assertEquals("[\"Houston\",\"we\",\"have\",\"a\",\"problem\"]", yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutputOfFString() {
     Object[] object = {new FString("Houston"), new FString("we"), new FString("have"),
         new FString("a"), new FString("problem")};
@@ -82,12 +97,14 @@ public class YailListTest extends TestCase {
     assertEquals("[\"Houston\",\"we\",\"have\",\"a\",\"problem\"]", yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutputOfNumber() {
     Object[] object = {new Integer(8), 9, 8.5};
     YailList yailList = YailList.makeList(object);
     assertEquals("[8,9,8.5]", yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutputOfHeterogenousItems() {
     Object[] firstList = {"Houston", "we", "have", "a", "problem"};
     Object[] secondList = {"China", "we", "have", "an", "ultimatum"};
@@ -104,6 +121,7 @@ public class YailListTest extends TestCase {
     assertEquals(correctOutput, yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutputOfDeepNestedList() {
     Object [] firstList = {"a"};
     Object [] secondList = {firstList, "b"};
@@ -114,6 +132,7 @@ public class YailListTest extends TestCase {
     assertEquals(correctOutput, yailList.toJSONString());
   }
 
+  @Test
   public void testJsonStringOutputOfNestedYailList() {
     Object [] firstList = {"a", "b"};
     Object [] secondList = {"a", "b"};
@@ -124,6 +143,7 @@ public class YailListTest extends TestCase {
     assertEquals(correctOutput, yailListTwo.toJSONString());
   }
 
+  @Test
   public void testCreationFromJavaList() {
     ArrayList<String> testList = new ArrayList<String>();
     testList.add("tom");
@@ -152,6 +172,7 @@ public class YailListTest extends TestCase {
     }
   }
 
+  @Test
   public void testCreationFromJavaCollection() {
     HashSet<String> testSet = new HashSet<String>();
     testSet.add("blind mouse #1");
@@ -180,6 +201,7 @@ public class YailListTest extends TestCase {
     }
   }
 
+  @Test
   public void testCreationFromArray() {
     String[] testArray = {"Mahmoud Ahmadinejad", "Alvin Stardust", "The Hamburglar"};
     YailList yailList = YailList.makeList(testArray);
@@ -205,6 +227,7 @@ public class YailListTest extends TestCase {
     }
   }
 
+  @Test
   public void testBigNumsInStringArray() {
     YailList list = YailList.makeList(new Object[] { IntNum.make(Long.MAX_VALUE), (Long) Long.MAX_VALUE });
     String[] strings = list.toStringArray();