From 7cc5e584d55583220d861d32bd82f6312dfd6005 Mon Sep 17 00:00:00 2001 From: Anthony Restaino Date: Sat, 10 Sep 2016 17:51:28 -0400 Subject: [PATCH] Add support for DuckDuckGo search suggestions, make improvements to google suggestions --- .../lightning/search/DuckSuggestionsTask.java | 199 ++++++++++++++++++ ...nsTask.java => GoogleSuggestionsTask.java} | 57 ++--- .../browser/lightning/search/Suggestions.java | 4 +- .../lightning/search/SuggestionsManager.java | 57 +++++ .../browser/lightning/utils/FileUtils.java | 14 ++ 5 files changed, 290 insertions(+), 41 deletions(-) create mode 100644 app/src/main/java/acr/browser/lightning/search/DuckSuggestionsTask.java rename app/src/main/java/acr/browser/lightning/search/{SuggestionsTask.java => GoogleSuggestionsTask.java} (78%) create mode 100644 app/src/main/java/acr/browser/lightning/search/SuggestionsManager.java diff --git a/app/src/main/java/acr/browser/lightning/search/DuckSuggestionsTask.java b/app/src/main/java/acr/browser/lightning/search/DuckSuggestionsTask.java new file mode 100644 index 0000000..34fd4a5 --- /dev/null +++ b/app/src/main/java/acr/browser/lightning/search/DuckSuggestionsTask.java @@ -0,0 +1,199 @@ +package acr.browser.lightning.search; + +import android.app.Application; +import android.content.Context; +import android.net.ConnectivityManager; +import android.net.NetworkInfo; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.text.TextUtils; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; + +import javax.net.ssl.HttpsURLConnection; + +import acr.browser.lightning.R; +import acr.browser.lightning.database.HistoryItem; +import acr.browser.lightning.utils.FileUtils; +import acr.browser.lightning.utils.Utils; + +/** + * Copyright 9/10/2016 Anthony Restaino + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public final class DuckSuggestionsTask { + + private static final String TAG = RetrieveSuggestionsTask.class.getSimpleName(); + + private static final Pattern SPACE_PATTERN = Pattern.compile(" ", Pattern.LITERAL); + private static final String ENCODING = "UTF-8"; + private static final long INTERVAL_DAY = TimeUnit.DAYS.toMillis(1); + private static final String DEFAULT_LANGUAGE = "en"; + @Nullable private static String sLanguage; + @NonNull private final SuggestionsResult mResultCallback; + @NonNull private final Application mApplication; + @NonNull private final String mSearchSubtitle; + @NonNull private String mQuery; + + DuckSuggestionsTask(@NonNull String query, + @NonNull Application application, + @NonNull SuggestionsResult callback) { + mQuery = query; + mResultCallback = callback; + mApplication = application; + mSearchSubtitle = mApplication.getString(R.string.suggestion); + } + + @NonNull + private static synchronized String getLanguage() { + if (sLanguage == null) { + sLanguage = Locale.getDefault().getLanguage(); + } + if (TextUtils.isEmpty(sLanguage)) { + sLanguage = DEFAULT_LANGUAGE; + } + return sLanguage; + } + + void run() { + List filter = new ArrayList<>(5); + try { + mQuery = SPACE_PATTERN.matcher(mQuery).replaceAll("+"); + mQuery = URLEncoder.encode(mQuery, ENCODING); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + File cache = downloadSuggestionsForQuery(mQuery, getLanguage(), mApplication); + if (!cache.exists()) { + post(filter); + return; + } + InputStream fileInput = null; + try { + fileInput = new FileInputStream(cache); + String content = FileUtils.readStringFromFile(fileInput, ENCODING); + JSONArray jsonArray = new JSONArray(content); + int counter = 0; + for (int n = 0, size = jsonArray.length(); n < size; n++) { + JSONObject object = jsonArray.getJSONObject(n); + String suggestion = object.getString("phrase"); + filter.add(new HistoryItem(mSearchSubtitle + " \"" + suggestion + '"', + suggestion, R.drawable.ic_search)); + counter++; + if (counter >= 5) { + break; + } + } + } catch (Exception e) { + post(filter); + e.printStackTrace(); + return; + } finally { + Utils.close(fileInput); + } + post(filter); + } + + private void post(@NonNull List result) { + mResultCallback.resultReceived(result); + } + + /** + * This method downloads the search suggestions for the specific query. + * NOTE: This is a blocking operation, do not run on the UI thread. + * + * @param query the query to get suggestions for + * @return the cache file containing the suggestions + */ + @NonNull + private static File downloadSuggestionsForQuery(@NonNull String query, String language, @NonNull Application app) { + File cacheFile = new File(app.getCacheDir(), query.hashCode() + Suggestions.CACHE_FILE_TYPE); + if (System.currentTimeMillis() - INTERVAL_DAY < cacheFile.lastModified()) { + return cacheFile; + } + if (!isNetworkConnected(app)) { + return cacheFile; + } + InputStream in = null; + FileOutputStream fos = null; + try { + URL url = new URL("https://duckduckgo.com/ac/?q=" + query); + HttpsURLConnection connection = (HttpsURLConnection) url.openConnection(); + connection.setDoInput(true); + connection.setRequestProperty("Accept-Encoding", "gzip"); + connection.connect(); + if (connection.getResponseCode() >= HttpURLConnection.HTTP_MULT_CHOICE || + connection.getResponseCode() < HttpURLConnection.HTTP_OK) { + Log.e(TAG, "Search API Responded with code: " + connection.getResponseCode()); + connection.disconnect(); + return cacheFile; + } + + in = connection.getInputStream(); + + if (in != null) { + in = new GZIPInputStream(in); + //noinspection IOResourceOpenedButNotSafelyClosed + fos = new FileOutputStream(cacheFile); + int buffer; + while ((buffer = in.read()) != -1) { + fos.write(buffer); + } + fos.flush(); + } + connection.disconnect(); + cacheFile.setLastModified(System.currentTimeMillis()); + } catch (Exception e) { + Log.w(TAG, "Problem getting search suggestions", e); + } finally { + Utils.close(in); + Utils.close(fos); + } + return cacheFile; + } + + private static boolean isNetworkConnected(@NonNull Context context) { + NetworkInfo networkInfo = getActiveNetworkInfo(context); + return networkInfo != null && networkInfo.isConnected(); + } + + @Nullable + private static NetworkInfo getActiveNetworkInfo(@NonNull Context context) { + ConnectivityManager connectivity = (ConnectivityManager) context + .getApplicationContext() + .getSystemService(Context.CONNECTIVITY_SERVICE); + if (connectivity == null) { + return null; + } + return connectivity.getActiveNetworkInfo(); + } + +} diff --git a/app/src/main/java/acr/browser/lightning/search/SuggestionsTask.java b/app/src/main/java/acr/browser/lightning/search/GoogleSuggestionsTask.java similarity index 78% rename from app/src/main/java/acr/browser/lightning/search/SuggestionsTask.java rename to app/src/main/java/acr/browser/lightning/search/GoogleSuggestionsTask.java index 01b1e3c..0f2d144 100644 --- a/app/src/main/java/acr/browser/lightning/search/SuggestionsTask.java +++ b/app/src/main/java/acr/browser/lightning/search/GoogleSuggestionsTask.java @@ -27,16 +27,15 @@ import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; + +import javax.net.ssl.HttpsURLConnection; import acr.browser.lightning.R; -import acr.browser.lightning.app.BrowserApp; import acr.browser.lightning.database.HistoryItem; -import com.anthonycr.bonsai.Action; -import com.anthonycr.bonsai.Observable; -import com.anthonycr.bonsai.Subscriber; import acr.browser.lightning.utils.Utils; -public class SuggestionsTask { +public class GoogleSuggestionsTask { private static final String TAG = RetrieveSuggestionsTask.class.getSimpleName(); @@ -51,32 +50,9 @@ public class SuggestionsTask { @NonNull private final String mSearchSubtitle; @NonNull private String mQuery; - private static volatile boolean sIsTaskExecuting = false; - - public static boolean isRequestInProgress() { - return sIsTaskExecuting; - } - - public static Observable> getObservable(@NonNull final String query, @NonNull final Context context) { - return Observable.create(new Action>() { - @Override - public void onSubscribe(@NonNull final Subscriber> subscriber) { - sIsTaskExecuting = true; - new SuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() { - @Override - public void resultReceived(@NonNull List searchResults) { - subscriber.onNext(searchResults); - subscriber.onComplete(); - } - }).run(); - sIsTaskExecuting = false; - } - }); - } - - private SuggestionsTask(@NonNull String query, - @NonNull Application application, - @NonNull SuggestionsResult callback) { + GoogleSuggestionsTask(@NonNull String query, + @NonNull Application application, + @NonNull SuggestionsResult callback) { mQuery = query; mResultCallback = callback; mApplication = application; @@ -104,11 +80,11 @@ public class SuggestionsTask { return sXpp; } - private void run() { + void run() { List filter = new ArrayList<>(5); try { mQuery = SPACE_PATTERN.matcher(mQuery).replaceAll("+"); - URLEncoder.encode(mQuery, ENCODING); + mQuery = URLEncoder.encode(mQuery, ENCODING); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } @@ -128,7 +104,7 @@ public class SuggestionsTask { if (eventType == XmlPullParser.START_TAG && "suggestion".equals(parser.getName())) { String suggestion = parser.getAttributeValue(null, "data"); filter.add(new HistoryItem(mSearchSubtitle + " \"" + suggestion + '"', - suggestion, R.drawable.ic_search)); + suggestion, R.drawable.ic_search)); counter++; if (counter >= 5) { break; @@ -137,6 +113,7 @@ public class SuggestionsTask { eventType = parser.next(); } } catch (Exception e) { + e.printStackTrace(); post(filter); return; } finally { @@ -171,12 +148,13 @@ public class SuggestionsTask { // Old API that doesn't support HTTPS // http://google.com/complete/search?q= + query + &output=toolbar&hl= + language URL url = new URL("https://suggestqueries.google.com/complete/search?output=toolbar&hl=" - + language + "&q=" + query); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + language + "&q=" + query); + HttpsURLConnection connection = (HttpsURLConnection) url.openConnection(); connection.setDoInput(true); + connection.setRequestProperty("Accept-Encoding", "gzip"); connection.connect(); if (connection.getResponseCode() >= HttpURLConnection.HTTP_MULT_CHOICE || - connection.getResponseCode() < HttpURLConnection.HTTP_OK) { + connection.getResponseCode() < HttpURLConnection.HTTP_OK) { Log.e(TAG, "Search API Responded with code: " + connection.getResponseCode()); connection.disconnect(); return cacheFile; @@ -185,6 +163,7 @@ public class SuggestionsTask { if (in != null) { //noinspection IOResourceOpenedButNotSafelyClosed + in = new GZIPInputStream(in); fos = new FileOutputStream(cacheFile); int buffer; while ((buffer = in.read()) != -1) { @@ -211,8 +190,8 @@ public class SuggestionsTask { @Nullable private static NetworkInfo getActiveNetworkInfo(@NonNull Context context) { ConnectivityManager connectivity = (ConnectivityManager) context - .getApplicationContext() - .getSystemService(Context.CONNECTIVITY_SERVICE); + .getApplicationContext() + .getSystemService(Context.CONNECTIVITY_SERVICE); if (connectivity == null) { return null; } diff --git a/app/src/main/java/acr/browser/lightning/search/Suggestions.java b/app/src/main/java/acr/browser/lightning/search/Suggestions.java index 92b15da..035772e 100644 --- a/app/src/main/java/acr/browser/lightning/search/Suggestions.java +++ b/app/src/main/java/acr/browser/lightning/search/Suggestions.java @@ -286,7 +286,7 @@ public class Suggestions extends BaseAdapter implements Filterable { @NonNull private Observable> getSuggestionsForQuery(@NonNull final String query) { - return SuggestionsTask.getObservable(query, mContext); + return SuggestionsManager.getObservable(query, mContext, SuggestionsManager.Source.GOOGLE); } @NonNull @@ -322,7 +322,7 @@ public class Suggestions extends BaseAdapter implements Filterable { } String query = constraint.toString().toLowerCase(Locale.getDefault()); - if (mSuggestions.shouldRequestNetwork() && !SuggestionsTask.isRequestInProgress()) { + if (mSuggestions.shouldRequestNetwork() && !SuggestionsManager.isRequestInProgress()) { mSuggestions.getSuggestionsForQuery(query) .subscribeOn(Schedulers.worker()) .observeOn(Schedulers.main()) diff --git a/app/src/main/java/acr/browser/lightning/search/SuggestionsManager.java b/app/src/main/java/acr/browser/lightning/search/SuggestionsManager.java new file mode 100644 index 0000000..9913b36 --- /dev/null +++ b/app/src/main/java/acr/browser/lightning/search/SuggestionsManager.java @@ -0,0 +1,57 @@ +package acr.browser.lightning.search; + +import android.content.Context; +import android.support.annotation.NonNull; + +import com.anthonycr.bonsai.Action; +import com.anthonycr.bonsai.Observable; +import com.anthonycr.bonsai.Subscriber; + +import java.util.List; + +import acr.browser.lightning.app.BrowserApp; +import acr.browser.lightning.database.HistoryItem; + +public class SuggestionsManager { + + public enum Source { + GOOGLE, + DUCK + } + + private static volatile boolean sIsTaskExecuting; + + public static boolean isRequestInProgress() { + return sIsTaskExecuting; + } + + public static Observable> getObservable(@NonNull final String query, @NonNull final Context context, @NonNull final Source source) { + return Observable.create(new Action>() { + @Override + public void onSubscribe(@NonNull final Subscriber> subscriber) { + sIsTaskExecuting = true; + switch (source) { + case GOOGLE: + new GoogleSuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() { + @Override + public void resultReceived(@NonNull List searchResults) { + subscriber.onNext(searchResults); + subscriber.onComplete(); + } + }).run(); + break; + case DUCK: + new DuckSuggestionsTask(query, BrowserApp.get(context), new SuggestionsResult() { + @Override + public void resultReceived(@NonNull List searchResults) { + subscriber.onNext(searchResults); + subscriber.onComplete(); + } + }).run(); + } + sIsTaskExecuting = false; + } + }); + } + +} diff --git a/app/src/main/java/acr/browser/lightning/utils/FileUtils.java b/app/src/main/java/acr/browser/lightning/utils/FileUtils.java index e5642ee..2f572ad 100644 --- a/app/src/main/java/acr/browser/lightning/utils/FileUtils.java +++ b/app/src/main/java/acr/browser/lightning/utils/FileUtils.java @@ -8,11 +8,14 @@ import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.util.Log; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintStream; import acr.browser.lightning.app.BrowserApp; @@ -134,4 +137,15 @@ public class FileUtils { } } + @NonNull + public static String readStringFromFile(@NonNull InputStream inputStream, @NonNull String encoding) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, encoding)); + StringBuilder result = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + result.append(line); + } + return result.toString(); + } + }