Add support for DuckDuckGo search suggestions, make improvements to google suggestions

app/src/main/java/acr/browser/lightning/search/DuckSuggestionsTask.java

@@ -0,0 +1,199 @@
+package acr.browser.lightning.search;
+
+import android.app.Application;
+import android.content.Context;
+import android.net.ConnectivityManager;
+import android.net.NetworkInfo;
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.text.TextUtils;
+import android.util.Log;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import javax.net.ssl.HttpsURLConnection;
+
+import acr.browser.lightning.R;
+import acr.browser.lightning.database.HistoryItem;
+import acr.browser.lightning.utils.FileUtils;
+import acr.browser.lightning.utils.Utils;
+
+/**
+ * Copyright 9/10/2016 Anthony Restaino
+ * <p/>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public final class DuckSuggestionsTask {
+
+    private static final String TAG = RetrieveSuggestionsTask.class.getSimpleName();
+
+    private static final Pattern SPACE_PATTERN = Pattern.compile(" ", Pattern.LITERAL);
+    private static final String ENCODING = "UTF-8";
+    private static final long INTERVAL_DAY = TimeUnit.DAYS.toMillis(1);
+    private static final String DEFAULT_LANGUAGE = "en";
+    @Nullable private static String sLanguage;
+    @NonNull private final SuggestionsResult mResultCallback;
+    @NonNull private final Application mApplication;
+    @NonNull private final String mSearchSubtitle;
+    @NonNull private String mQuery;
+
+    DuckSuggestionsTask(@NonNull String query,
+                        @NonNull Application application,
+                        @NonNull SuggestionsResult callback) {
+        mQuery = query;
+        mResultCallback = callback;
+        mApplication = application;
+        mSearchSubtitle = mApplication.getString(R.string.suggestion);
+    }
+
+    @NonNull
+    private static synchronized String getLanguage() {
+        if (sLanguage == null) {
+            sLanguage = Locale.getDefault().getLanguage();
+        }
+        if (TextUtils.isEmpty(sLanguage)) {
+            sLanguage = DEFAULT_LANGUAGE;
+        }
+        return sLanguage;
+    }
+
+    void run() {
+        List<HistoryItem> filter = new ArrayList<>(5);
+        try {
+            mQuery = SPACE_PATTERN.matcher(mQuery).replaceAll("+");
+            mQuery = URLEncoder.encode(mQuery, ENCODING);
+        } catch (UnsupportedEncodingException e) {
+            e.printStackTrace();
+        }
+        File cache = downloadSuggestionsForQuery(mQuery, getLanguage(), mApplication);
+        if (!cache.exists()) {
+            post(filter);
+            return;
+        }
+        InputStream fileInput = null;
+        try {
+            fileInput = new FileInputStream(cache);
+            String content = FileUtils.readStringFromFile(fileInput, ENCODING);
+            JSONArray jsonArray = new JSONArray(content);
+            int counter = 0;
+            for (int n = 0, size = jsonArray.length(); n < size; n++) {
+                JSONObject object = jsonArray.getJSONObject(n);
+                String suggestion = object.getString("phrase");
+                filter.add(new HistoryItem(mSearchSubtitle + " \"" + suggestion + '"',
+                    suggestion, R.drawable.ic_search));
+                counter++;
+                if (counter >= 5) {
+                    break;
+                }
+            }
+        } catch (Exception e) {
+            post(filter);
+            e.printStackTrace();
+            return;
+        } finally {
+            Utils.close(fileInput);
+        }
+        post(filter);
+    }
+
+    private void post(@NonNull List<HistoryItem> result) {
+        mResultCallback.resultReceived(result);
+    }
+
+    /**
+     * This method downloads the search suggestions for the specific query.
+     * NOTE: This is a blocking operation, do not run on the UI thread.
+     *
+     * @param query the query to get suggestions for
+     * @return the cache file containing the suggestions
+     */
+    @NonNull
+    private static File downloadSuggestionsForQuery(@NonNull String query, String language, @NonNull Application app) {
+        File cacheFile = new File(app.getCacheDir(), query.hashCode() + Suggestions.CACHE_FILE_TYPE);
+        if (System.currentTimeMillis() - INTERVAL_DAY < cacheFile.lastModified()) {
+            return cacheFile;
+        }
+        if (!isNetworkConnected(app)) {
+            return cacheFile;
+        }
+        InputStream in = null;
+        FileOutputStream fos = null;
+        try {
+            URL url = new URL("https://duckduckgo.com/ac/?q=" + query);
+            HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
+            connection.setDoInput(true);
+            connection.setRequestProperty("Accept-Encoding", "gzip");
+            connection.connect();
+            if (connection.getResponseCode() >= HttpURLConnection.HTTP_MULT_CHOICE ||
+                connection.getResponseCode() < HttpURLConnection.HTTP_OK) {
+                Log.e(TAG, "Search API Responded with code: " + connection.getResponseCode());
+                connection.disconnect();
+                return cacheFile;
+            }
+
+            in = connection.getInputStream();
+
+            if (in != null) {
+                in = new GZIPInputStream(in);
+                //noinspection IOResourceOpenedButNotSafelyClosed
+                fos = new FileOutputStream(cacheFile);
+                int buffer;
+                while ((buffer = in.read()) != -1) {
+                    fos.write(buffer);
+                }
+                fos.flush();
+            }
+            connection.disconnect();
+            cacheFile.setLastModified(System.currentTimeMillis());
+        } catch (Exception e) {
+            Log.w(TAG, "Problem getting search suggestions", e);
+        } finally {
+            Utils.close(in);
+            Utils.close(fos);
+        }
+        return cacheFile;
+    }
+
+    private static boolean isNetworkConnected(@NonNull Context context) {
+        NetworkInfo networkInfo = getActiveNetworkInfo(context);
+        return networkInfo != null && networkInfo.isConnected();
+    }
+
+    @Nullable
+    private static NetworkInfo getActiveNetworkInfo(@NonNull Context context) {
+        ConnectivityManager connectivity = (ConnectivityManager) context
+            .getApplicationContext()
+            .getSystemService(Context.CONNECTIVITY_SERVICE);
+        if (connectivity == null) {
+            return null;
+        }
+        return connectivity.getActiveNetworkInfo();
+    }
+
+}

app/src/main/java/acr/browser/lightning/search/GoogleSuggestionsTask.java

@@ -27,16 +27,15 @@ import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import javax.net.ssl.HttpsURLConnection;
 
 import acr.browser.lightning.R;
-import acr.browser.lightning.app.BrowserApp;
 import acr.browser.lightning.database.HistoryItem;
-import com.anthonycr.bonsai.Action;
-import com.anthonycr.bonsai.Observable;
-import com.anthonycr.bonsai.Subscriber;
 import acr.browser.lightning.utils.Utils;
 
-public class SuggestionsTask {
+public class GoogleSuggestionsTask {
 
     private static final String TAG = RetrieveSuggestionsTask.class.getSimpleName();
 
@@ -51,32 +50,9 @@ public class SuggestionsTask {
     @NonNull private final String mSearchSubtitle;
     @NonNull private String mQuery;
 
-    private static volatile boolean sIsTaskExecuting = false;
+    GoogleSuggestionsTask(@NonNull String query,
-
+                          @NonNull Application application,
-    public static boolean isRequestInProgress() {
+                          @NonNull SuggestionsResult callback) {
-        return sIsTaskExecuting;
-    }
-
-    public static Observable<List<HistoryItem>> getObservable(@NonNull final String query, @NonNull final Context context) {
-        return Observable.create(new Action<List<HistoryItem>>() {
-            @Override
-            public void onSubscribe(@NonNull final Subscriber<List<HistoryItem>> subscriber) {
-                sIsTaskExecuting = true;
-                new SuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() {
-                    @Override
-                    public void resultReceived(@NonNull List<HistoryItem> searchResults) {
-                        subscriber.onNext(searchResults);
-                        subscriber.onComplete();
-                    }
-                }).run();
-                sIsTaskExecuting = false;
-            }
-        });
-    }
-
-    private SuggestionsTask(@NonNull String query,
-                            @NonNull Application application,
-                            @NonNull SuggestionsResult callback) {
         mQuery = query;
         mResultCallback = callback;
         mApplication = application;
@@ -104,11 +80,11 @@ public class SuggestionsTask {
         return sXpp;
     }
 
-    private void run() {
+    void run() {
         List<HistoryItem> filter = new ArrayList<>(5);
         try {
             mQuery = SPACE_PATTERN.matcher(mQuery).replaceAll("+");
-            URLEncoder.encode(mQuery, ENCODING);
+            mQuery = URLEncoder.encode(mQuery, ENCODING);
         } catch (UnsupportedEncodingException e) {
             e.printStackTrace();
         }
@@ -128,7 +104,7 @@ public class SuggestionsTask {
                 if (eventType == XmlPullParser.START_TAG && "suggestion".equals(parser.getName())) {
                     String suggestion = parser.getAttributeValue(null, "data");
                     filter.add(new HistoryItem(mSearchSubtitle + " \"" + suggestion + '"',
-                            suggestion, R.drawable.ic_search));
+                        suggestion, R.drawable.ic_search));
                     counter++;
                     if (counter >= 5) {
                         break;
@@ -137,6 +113,7 @@ public class SuggestionsTask {
                 eventType = parser.next();
             }
         } catch (Exception e) {
+            e.printStackTrace();
             post(filter);
             return;
         } finally {
@@ -171,12 +148,13 @@ public class SuggestionsTask {
             // Old API that doesn't support HTTPS
             // http://google.com/complete/search?q= + query + &output=toolbar&hl= + language
             URL url = new URL("https://suggestqueries.google.com/complete/search?output=toolbar&hl="
-                    + language + "&q=" + query);
+                + language + "&q=" + query);
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
             connection.setDoInput(true);
+            connection.setRequestProperty("Accept-Encoding", "gzip");
             connection.connect();
             if (connection.getResponseCode() >= HttpURLConnection.HTTP_MULT_CHOICE ||
-                    connection.getResponseCode() < HttpURLConnection.HTTP_OK) {
+                connection.getResponseCode() < HttpURLConnection.HTTP_OK) {
                 Log.e(TAG, "Search API Responded with code: " + connection.getResponseCode());
                 connection.disconnect();
                 return cacheFile;
@@ -185,6 +163,7 @@ public class SuggestionsTask {
 
             if (in != null) {
                 //noinspection IOResourceOpenedButNotSafelyClosed
+                in = new GZIPInputStream(in);
                 fos = new FileOutputStream(cacheFile);
                 int buffer;
                 while ((buffer = in.read()) != -1) {
@@ -211,8 +190,8 @@ public class SuggestionsTask {
     @Nullable
     private static NetworkInfo getActiveNetworkInfo(@NonNull Context context) {
         ConnectivityManager connectivity = (ConnectivityManager) context
-                .getApplicationContext()
+            .getApplicationContext()
-                .getSystemService(Context.CONNECTIVITY_SERVICE);
+            .getSystemService(Context.CONNECTIVITY_SERVICE);
         if (connectivity == null) {
             return null;
         }

app/src/main/java/acr/browser/lightning/search/Suggestions.java

@@ -286,7 +286,7 @@ public class Suggestions extends BaseAdapter implements Filterable {
 
     @NonNull
     private Observable<List<HistoryItem>> getSuggestionsForQuery(@NonNull final String query) {
-        return SuggestionsTask.getObservable(query, mContext);
+        return SuggestionsManager.getObservable(query, mContext, SuggestionsManager.Source.GOOGLE);
     }
 
     @NonNull
@@ -322,7 +322,7 @@ public class Suggestions extends BaseAdapter implements Filterable {
             }
             String query = constraint.toString().toLowerCase(Locale.getDefault());
 
-            if (mSuggestions.shouldRequestNetwork() && !SuggestionsTask.isRequestInProgress()) {
+            if (mSuggestions.shouldRequestNetwork() && !SuggestionsManager.isRequestInProgress()) {
                 mSuggestions.getSuggestionsForQuery(query)
                     .subscribeOn(Schedulers.worker())
                     .observeOn(Schedulers.main())

app/src/main/java/acr/browser/lightning/search/SuggestionsManager.java

@@ -0,0 +1,57 @@
+package acr.browser.lightning.search;
+
+import android.content.Context;
+import android.support.annotation.NonNull;
+
+import com.anthonycr.bonsai.Action;
+import com.anthonycr.bonsai.Observable;
+import com.anthonycr.bonsai.Subscriber;
+
+import java.util.List;
+
+import acr.browser.lightning.app.BrowserApp;
+import acr.browser.lightning.database.HistoryItem;
+
+public class SuggestionsManager {
+
+    public enum Source {
+        GOOGLE,
+        DUCK
+    }
+
+    private static volatile boolean sIsTaskExecuting;
+
+    public static boolean isRequestInProgress() {
+        return sIsTaskExecuting;
+    }
+
+    public static Observable<List<HistoryItem>> getObservable(@NonNull final String query, @NonNull final Context context, @NonNull final Source source) {
+        return Observable.create(new Action<List<HistoryItem>>() {
+            @Override
+            public void onSubscribe(@NonNull final Subscriber<List<HistoryItem>> subscriber) {
+                sIsTaskExecuting = true;
+                switch (source) {
+                    case GOOGLE:
+                        new GoogleSuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() {
+                            @Override
+                            public void resultReceived(@NonNull List<HistoryItem> searchResults) {
+                                subscriber.onNext(searchResults);
+                                subscriber.onComplete();
+                            }
+                        }).run();
+                        break;
+                    case DUCK:
+                        new DuckSuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() {
+                            @Override
+                            public void resultReceived(@NonNull List<HistoryItem> searchResults) {
+                                subscriber.onNext(searchResults);
+                                subscriber.onComplete();
+                            }
+                        }).run();
+                }
+                sIsTaskExecuting = false;
+            }
+        });
+    }
+
+}

app/src/main/java/acr/browser/lightning/utils/FileUtils.java

@@ -8,11 +8,14 @@ import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
 import android.util.Log;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.PrintStream;
 
 import acr.browser.lightning.app.BrowserApp;
@@ -134,4 +137,15 @@ public class FileUtils {
         }
     }
 
+    @NonNull
+    public static String readStringFromFile(@NonNull InputStream inputStream, @NonNull String encoding) throws IOException {
+        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, encoding));
+        StringBuilder result = new StringBuilder();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            result.append(line);
+        }
+        return result.toString();
+    }
+
 }