Reduce logging and improve static files loading

Reduce logging so only anomalous situations result in log entries (you
can turn logging back on by setting appinventor.debugging to true in
appengine-web.xml).

Disable App Stats, as it really isn’t needed in production.

Provide reasonable caching and expiration on some static files,
including making sure they are served statically (this is required when
removing App Stats).

Change-Id: I84bd4877a2cbeba79f7da9248919920ea27c499b

appinventor/appengine/src/com/google/appinventor/server/GallerySearchIndex.java

@@ -20,6 +20,7 @@ import com.google.appengine.api.search.ScoredDocument;
 import com.google.appengine.api.search.SearchException;
 import com.google.appengine.api.search.SearchServiceFactory;
 import com.google.appengine.api.search.StatusCode;
+import com.google.appinventor.server.flags.Flag;
 import com.google.appinventor.server.storage.GalleryStorageIo;
 import com.google.appinventor.server.storage.GalleryStorageIoInstanceHolder;
 import com.google.appinventor.shared.rpc.project.GalleryApp;
@@ -42,6 +43,8 @@ public class GallerySearchIndex {
   private static volatile GallerySearchIndex  instance= null;
   private static final int SEARCH_RETRY_MAX = 3;
   private final int NUMBER_FOUND_ACCURACY = 100;
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
+
   /**
    * The default constructor of GallerySearchIndex
    */
@@ -128,7 +131,9 @@ public class GallerySearchIndex {
     boolean retry = true;
     while (retry){
       try {
-        LOG.info("Sending query " + query);
+        if (DEBUG) {
+          LOG.info("Sending query " + query);
+        }
         results = getIndex().search(query);
         // search successful
         retry = false;
@@ -151,7 +156,9 @@ public class GallerySearchIndex {
     if (results != null){
       // Iterate over the documents in the results
       for (ScoredDocument document : results) {
-        LOG.info("Find:" + document.getId());
+        if (DEBUG) {
+          LOG.info("Find:" + document.getId());
+        }
       }
 
       // Iterate over the documents in the results

appinventor/appengine/src/com/google/appinventor/server/GalleryServlet.java

@@ -26,6 +26,7 @@ import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.appengine.tools.cloudstorage.GcsOutputChannel;
 import com.google.appengine.tools.cloudstorage.GcsService;
 import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.appinventor.server.flags.Flag;
 import com.google.appinventor.server.util.CacheHeaders;
 import com.google.appinventor.server.util.CacheHeadersImpl;
 import com.google.appinventor.shared.rpc.ServerLayout;
@@ -70,7 +71,7 @@ public class GalleryServlet extends OdeServlet {
   private static final String CONTENT_TYPE = "text/html; charset=utf-8";
 
   private final GalleryService galleryService = new GalleryServiceImpl();
-
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
 
   @Override
   public void doPost(HttpServletRequest req, HttpServletResponse resp) {
@@ -83,8 +84,10 @@ public class GalleryServlet extends OdeServlet {
 
     if (true) {
       String requestType = uriComponents[REQUEST_TYPE_INDEX];
-      LOG.info("######### GOT IN URI");
-      LOG.info(requestType);
+      if (DEBUG) {
+        LOG.info("######### GOT IN URI");
+        LOG.info(requestType);
+      }
 
       long project_Id = -1;
       String user_Id = "-1";

appinventor/appengine/src/com/google/appinventor/server/LoginServlet.java

@@ -73,6 +73,7 @@ public class LoginServlet extends HttpServlet {
   private static final Flag<Boolean> useLocal = Flag.createFlag("auth.uselocal", false);
   private static final UserService userService = UserServiceFactory.getUserService();
   private final PolicyFactory sanitizer = new HtmlPolicyBuilder().allowElements("p").toFactory();
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
 
   public void init(ServletConfig config) throws ServletException {
     super.init(config);
@@ -83,7 +84,9 @@ public class LoginServlet extends HttpServlet {
 
     PrintWriter out;
     String [] components = req.getRequestURI().split("/");
-    LOG.info("requestURI = " + req.getRequestURI());
+    if (DEBUG) {
+      LOG.info("requestURI = " + req.getRequestURI());
+    }
     String page = getPage(req);
 
     OdeAuthFilter.UserInfo userInfo = OdeAuthFilter.getUserInfo(req);
@@ -100,7 +103,9 @@ public class LoginServlet extends HttpServlet {
     String galleryId = params.get("galleryId");
     String redirect = params.get("redirect");
 
-    LOG.info("locale = " + locale + " bundle: " + new Locale(locale));
+    if (DEBUG) {
+      LOG.info("locale = " + locale + " bundle: " + new Locale(locale));
+    }
     ResourceBundle bundle = ResourceBundle.getBundle("com/google/appinventor/server/loginmessages", new Locale(locale));
 
     if (page.equals("google")) {
@@ -124,7 +129,9 @@ public class LoginServlet extends HttpServlet {
       }
 
       String newCookie = userInfo.buildCookie(false);
-      LOG.info("newCookie = " + newCookie);
+      if (DEBUG) {
+        LOG.info("newCookie = " + newCookie);
+      }
       if (newCookie != null) {
         Cookie cook = new Cookie("AppInventor", newCookie);
         cook.setPath("/");
@@ -179,7 +186,9 @@ public class LoginServlet extends HttpServlet {
         fail(req, resp, "Invalid Set Password Link");
         return;
       }
-      LOG.info("setpw email = " + data.email);
+      if (DEBUG) {
+        LOG.info("setpw email = " + data.email);
+      }
       User user = storageIo.getUserFromEmail(data.email);
       userInfo = new OdeAuthFilter.UserInfo(); // Create new userInfo object
       userInfo.setUserId(user.getUserId()); // This effectively logs us in!
@@ -275,7 +284,9 @@ public class LoginServlet extends HttpServlet {
 
     ResourceBundle bundle = ResourceBundle.getBundle("com/google/appinventor/server/loginmessages", new Locale(locale));
 
-    LOG.info("locale = " + locale + " bundle: " + new Locale(locale));
+    if (DEBUG) {
+      LOG.info("locale = " + locale + " bundle: " + new Locale(locale));
+    }
     if (page.equals("sendlink")) {
       String email = params.get("email");
       if (email == null) {
@@ -346,11 +357,15 @@ public class LoginServlet extends HttpServlet {
       return;
     }
 
-    LOG.info("userInfo = " + userInfo + " user = " + user);
+    if (DEBUG) {
+      LOG.info("userInfo = " + userInfo + " user = " + user);
+    }
     userInfo.setUserId(user.getUserId());
     userInfo.setIsAdmin(user.getIsAdmin());
     String newCookie = userInfo.buildCookie(false);
-    LOG.info("newCookie = " + newCookie);
+    if (DEBUG) {
+      LOG.info("newCookie = " + newCookie);
+    }
     if (newCookie != null) {
       Cookie cook = new Cookie("AppInventor", newCookie);
       cook.setPath("/");

appinventor/appengine/src/com/google/appinventor/server/OdeAuthFilter.java

@@ -68,6 +68,7 @@ public class OdeAuthFilter implements Filter {
   static final Flag<Integer> renewTime = Flag.createFlag("session.renew", 30);
 
   private final LocalUser localUser = LocalUser.getInstance();
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
 
   /**
    * Filters using Google Accounts
@@ -100,14 +101,18 @@ public class OdeAuthFilter implements Filter {
     // Use Local Authentication
     UserInfo userInfo = getUserInfo(httpRequest);
     if (userInfo == null) {        // Invalid Login
-      LOG.info("uinfo is null on login.");
+      if (DEBUG) {
+        LOG.info("uinfo is null on login.");
+      }
       // If the URI starts with /ode, then we are being invoked through
       // the App Inventor client. In that case we are in an XMLHttpRequest
       // (aka ajax) so we cannot send a redirect to the login page
       // instead we return SC_PRECONDITION_FAILED which tips off the
       // client that it needs to reload itself to the login page.
       String uri = httpRequest.getRequestURI();
-      LOG.info("Not Logged In: uri = " + uri);
+      if (DEBUG) {
+        LOG.info("Not Logged In: uri = " + uri);
+      }
       if (uri.startsWith("/ode")) {
         httpResponse.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED);
       } else {
@@ -164,7 +169,9 @@ public class OdeAuthFilter implements Filter {
       String newCookie = userInfo.buildCookie(true);
       if (newCookie != null) {  // If we get a value here, it is time to renew
                                 // the Cookie
-        LOG.info("Renewing the authentication Cookie");
+        if (DEBUG) {
+          LOG.info("Renewing the authentication Cookie");
+        }
         Cookie cook = new Cookie("AppInventor", newCookie);
         cook.setPath("/");
         response.addCookie(cook);
@@ -330,7 +337,9 @@ public class OdeAuthFilter implements Filter {
         for (Cookie cookie : cookies) {
           if ("AppInventor".equals(cookie.getName())) {
             String rawData = cookie.getValue();
-            LOG.info("getUserInfo: rawCookie = " + rawData);
+            if (DEBUG) {
+              LOG.info("getUserInfo: rawCookie = " + rawData);
+            }
             Crypter crypter = getCrypter();
             CookieAuth.cookie cookieToken = CookieAuth.cookie.parseFrom(
               crypter.decrypt(Base64Coder.decode(rawData)));

appinventor/appengine/src/com/google/appinventor/server/ProjectServiceImpl.java

@@ -64,6 +64,8 @@ public class ProjectServiceImpl extends OdeRemoteServiceServlet implements Proje
   private final transient YoungAndroidProjectService youngAndroidProject =
       new YoungAndroidProjectService(storageIo);
 
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
+
   /**
    * Creates a new project.
    * @param projectType  type of new project
@@ -601,7 +603,9 @@ public class ProjectServiceImpl extends OdeRemoteServiceServlet implements Proje
       GcsService fileService = GcsServiceFactory.createGcsService();
       GcsFilename readableFile = new GcsFilename(Flag.createFlag("gallery.bucket", "").get(), galleryPath);
       GcsInputChannel readChannel = fileService.openPrefetchingReadChannel(readableFile, 0, 16384);
-      LOG.log(Level.INFO, "#### in newProjectFromGallery, past readChannel");
+      if (DEBUG) {
+        LOG.log(Level.INFO, "#### in newProjectFromGallery, past readChannel");
+      }
       InputStream gcsis = Channels.newInputStream(readChannel);
       // ok, we don't want to send the gcs stream because it can time out as we
       // process the zip. We need to copy to a byte buffer first, then send a bytestream
@@ -615,7 +619,9 @@ public class ProjectServiceImpl extends OdeRemoteServiceServlet implements Proje
       }
 
       InputStream bais = new ByteArrayInputStream(bao.toByteArray());
-      LOG.log(Level.INFO, "#### in newProjectFromGallery, past newInputStream");
+      if (DEBUG) {
+        LOG.log(Level.INFO, "#### in newProjectFromGallery, past newInputStream");
+      }
 
       // close the gcs
       readChannel.close();
@@ -623,7 +629,9 @@ public class ProjectServiceImpl extends OdeRemoteServiceServlet implements Proje
       FileImporter fileImporter = new FileImporterImpl();
       UserProject userProject = fileImporter.importProject(userInfoProvider.getUserId(),
         projectName, bais);
-      LOG.log(Level.INFO, "#### in newProjectFromGallery, past importProject");
+      if (DEBUG) {
+        LOG.log(Level.INFO, "#### in newProjectFromGallery, past importProject");
+      }
 
       // set the attribution id of the project
       storageIo.setProjectAttributionId(userInfoProvider.getUserId(), userProject.getProjectId(),galleryId);
@@ -655,15 +663,12 @@ public class ProjectServiceImpl extends OdeRemoteServiceServlet implements Proje
 
   private void validateSessionId(String sessionId) throws InvalidSessionException {
     String storedSessionId = userInfoProvider.getSessionId();
-    if (storedSessionId == null) {
-      LOG.info("storedSessionId is null");
-    } else {
-      LOG.info("storedSessionId = " + storedSessionId);
-    }
-    if (sessionId == null) {
-      LOG.info("sessionId is null");
-    } else {
-      LOG.info("sessionId = " + sessionId);
+    if (DEBUG) {
+      if (storedSessionId == null) {
+        LOG.info("storedSessionId is null");
+      } else {
+        LOG.info("storedSessionId = " + storedSessionId);
+      }
     }
     if (sessionId.equals("force")) { // If we are forcing our way -- no check
       return;

appinventor/appengine/src/com/google/appinventor/server/project/youngandroid/YoungAndroidProjectService.java

@@ -119,6 +119,7 @@ public final class YoungAndroidProjectService extends CommonProjectService {
   // host[:port] to tell build server app host url
   private static final Flag<String> appengineHost =
       Flag.createFlag("appengine.host", "");
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
 
   public YoungAndroidProjectService(StorageIo storageIo) {
     super(YoungAndroidProjectNode.YOUNG_ANDROID_PROJECT_TYPE, storageIo);
@@ -777,7 +778,9 @@ public final class YoungAndroidProjectService extends CommonProjectService {
           try {
             String content = readContent(connection.getInputStream());
             if (content != null && !content.isEmpty()) {
-              LOG.info("The current progress is " + content + "%.");
+              if (DEBUG) {
+                LOG.info("The current progress is " + content + "%.");
+              }
               currentProgress = Integer.parseInt(content);
             }
           } catch (IOException e) {

appinventor/appengine/src/com/google/appinventor/server/storage/ObjectifyStorageIo.java

@@ -129,6 +129,7 @@ public class ObjectifyStorageIo implements  StorageIo {
   private final boolean useGcs = Flag.createFlag("use.gcs", true).get();
 
   private final boolean conversionEnabled = false; // We are converting GCS <=> Blobstore
+  private static final boolean DEBUG = Flag.createFlag("appinventor.debugging", false).get();
 
   // Use this class to define the work of a job that can be
   // retried. The "datastore" argument to run() is the Objectify
@@ -217,12 +218,14 @@ public class ObjectifyStorageIo implements  StorageIo {
     RetryParams retryParams = new RetryParams.Builder().initialRetryDelayMillis(100)
       .retryMaxAttempts(10)
       .totalRetryPeriodMillis(10000).build();
-    LOG.log(Level.INFO, "RetryParams: getInitialRetryDelayMillis() = " + retryParams.getInitialRetryDelayMillis());
-    LOG.log(Level.INFO, "RetryParams: getRequestTimeoutMillis() = " + retryParams.getRequestTimeoutMillis());
-    LOG.log(Level.INFO, "RetryParams: getRetryDelayBackoffFactor() = " + retryParams.getRetryDelayBackoffFactor());
-    LOG.log(Level.INFO, "RetryParams: getRetryMaxAttempts() = " + retryParams.getRetryMaxAttempts());
-    LOG.log(Level.INFO, "RetryParams: getRetryMinAttempts() = " + retryParams.getRetryMinAttempts());
-    LOG.log(Level.INFO, "RetryParams: getTotalRetryPeriodMillis() = " + retryParams.getTotalRetryPeriodMillis());
+    if (DEBUG) {
+      LOG.log(Level.INFO, "RetryParams: getInitialRetryDelayMillis() = " + retryParams.getInitialRetryDelayMillis());
+      LOG.log(Level.INFO, "RetryParams: getRequestTimeoutMillis() = " + retryParams.getRequestTimeoutMillis());
+      LOG.log(Level.INFO, "RetryParams: getRetryDelayBackoffFactor() = " + retryParams.getRetryDelayBackoffFactor());
+      LOG.log(Level.INFO, "RetryParams: getRetryMaxAttempts() = " + retryParams.getRetryMaxAttempts());
+      LOG.log(Level.INFO, "RetryParams: getRetryMinAttempts() = " + retryParams.getRetryMinAttempts());
+      LOG.log(Level.INFO, "RetryParams: getTotalRetryPeriodMillis() = " + retryParams.getTotalRetryPeriodMillis());
+    }
     gcsService = GcsServiceFactory.createGcsService(retryParams);
     memcache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
     initMotd();
@@ -1541,7 +1544,9 @@ public class ObjectifyStorageIo implements  StorageIo {
           if (fd == null) {
             fd = datastore.find(projectFileKey(projectKey(projectId), fileName));
           } else {
-            LOG.log(Level.INFO, "Fetched " + key.getString() + " from memcache.");
+            if (DEBUG) {
+              LOG.log(Level.INFO, "Fetched " + key.getString() + " from memcache.");
+            }
           }
 
           // <Screen>.yail files are missing when user converts AI1 project to AI2
@@ -1821,7 +1826,9 @@ public class ObjectifyStorageIo implements  StorageIo {
                 while (bytesRead < fileSize) {
                   bytesRead += readChannel.read(resultBuffer);
                   if (bytesRead < fileSize) {
-                    LOG.log(Level.INFO, "readChannel: bytesRead = " + bytesRead + " fileSize = " + fileSize);
+                    if (DEBUG) {
+                      LOG.log(Level.INFO, "readChannel: bytesRead = " + bytesRead + " fileSize = " + fileSize);
+                    }
                   }
                 }
                 recovered = true;
@@ -2051,7 +2058,9 @@ public class ObjectifyStorageIo implements  StorageIo {
                   while (bytesRead < fileSize) {
                     bytesRead += readChannel.read(resultBuffer);
                     if (bytesRead < fileSize) {
-                      LOG.log(Level.INFO, "readChannel: bytesRead = " + bytesRead + " fileSize = " + fileSize);
+                      if (DEBUG) {
+                        LOG.log(Level.INFO, "readChannel: bytesRead = " + bytesRead + " fileSize = " + fileSize);
+                      }
                     }
                   }
                   recovered = true;

appinventor/appengine/war/WEB-INF/appengine-web.xml

@@ -13,6 +13,16 @@
     </include>
     <include path="/media/*" expiration="365d" />
     <include path="/images/*" expiration="365d" />
+    <include path="/Ya_tos_form.html" expiration="1d" />
+    <include path="/favicon.ico" expiration="365d" />
+    <include path="/gwt.css" expiration="1d" />
+    <include path="/Ya.css" expiration="1d" />
+    <include path="/templates/**" expiration="1d" />
+    <include path="/closure-library/closure/goog/base.js" expiration="1d" />
+    <include path="/closure-library/closure/goog/css/dialog.css" expiration="1d" />
+    <include path="/closure-library/closure/goog/deps.js" expiration="1d" />
+    <include path="/robots.txt" expiration="30d" />
+
     <!-- These are not cached on purpose -->
     <include path="/ode/**.nocache.*" expiration="0s" />
     <include path="/blocklyframe.html" expiration="0s" />

appinventor/appengine/war/WEB-INF/web.xml

@@ -18,6 +18,7 @@
        explicitly list all the ones we do want, which seems
        error-prone.
     -->
+  <!-- App Stats Disabled
   <filter>
     <filter-name>appstats</filter-name>
     <filter-class>com.google.appengine.tools.appstats.AppstatsFilter</filter-class>
@@ -30,7 +31,7 @@
     <filter-name>appstats</filter-name>
     <url-pattern>/*</url-pattern>
   </filter-mapping>
-
+-->
 
   <!-- Security constraints: require login for these urls -->
   <!-- This is how we implement Google Account login -->