filtering of Searching & TopTrends, checkboxes, autoupdating of TopTrends in 2 min, not filtering retwists of our own posts, localization for tr

js/interface_home.js

@@ -67,22 +67,6 @@ var InterfaceFunctions = function()
             $(".dropdown-menu-following").attr("href","#");
             $(".dropdown-menu-following").bind("click", function()
             { alert(polyglot.t("You are not following anyone because you are not logged in."))} );
-            twisterRpc("gettrendinghashtags", [10],
-                                function(args, ret) {
-                                    for( var i = 0; i < ret.length; i++ ) {
-
-                                       var $li = $("<li>");
-                                       var hashtagLinkTemplate = $("#hashtag-link-template").clone(true);
-                                       hashtagLinkTemplate.removeAttr("id");
-                                       hashtagLinkTemplate.attr("href",$.MAL.hashtagUrl(ret[i]));
-                                       hashtagLinkTemplate.text("#"+ret[i]);
-                                       $li.append(hashtagLinkTemplate);
-                                       $(".toptrends-list").append($li);
-                                    }
-                                }, {},
-                                function(args, ret) {
-                                   console.log("Error with gettrendinghashtags. Older twister daemon?");
-                                }, {});
         }
         else
         {
@@ -119,23 +103,6 @@ var InterfaceFunctions = function()
                      setTimeout("getRandomFollowSuggestion(processSuggestion)", 1000);
                      setTimeout("getRandomFollowSuggestion(processSuggestion)", 1000);
 
-                     twisterRpc("gettrendinghashtags", [10],
-                                function(args, ret) {
-                                    for( var i = 0; i < ret.length; i++ ) {
-
-                                       var $li = $("<li>");
-                                       var hashtagLinkTemplate = $("#hashtag-link-template").clone(true);
-                                       hashtagLinkTemplate.removeAttr("id");
-                                       hashtagLinkTemplate.attr("href",$.MAL.hashtagUrl(ret[i]));
-                                       hashtagLinkTemplate.text("#"+ret[i]);
-                                       $li.append(hashtagLinkTemplate);
-                                       $(".toptrends-list").append($li);
-                                    }
-                                }, {},
-                                function(args, ret) {
-                                   console.log("Error with gettrendinghashtags. Older twister daemon?");
-                                }, {});
-
                      if( args.cbFunc )
                         args.cbFunc(args.cbArg);
                  }, {cbFunc:cbFunc, cbArg:cbArg});
@@ -154,9 +121,47 @@ var InterfaceFunctions = function()
                     });
                 });
         }
+
+        setTimeout(updateTrendingHashtags, 1000);
+        setInterval(updateTrendingHashtags, 120*1000); // FIXME should be an option for this
     }
 };
 
+function updateTrendingHashtags() {
+    twisterRpc('gettrendinghashtags', [10],
+        function(args, ret) {
+            $('.toptrends-list').empty();
+            //console.log('hashtags trends: '+ret);
+            for( var i = 0; i < ret.length; i++ ) {
+                if ($.Options.getFilterLangForTopTrendsOpt())
+                    var langFilterData = filterLang(ret[i]);
+                if (!$.Options.getFilterLangForTopTrendsOpt() || langFilterData['pass'] || $.Options.getFilterLangSimulateOpt()) {
+                    var $li = $('<li>');
+                    var hashtagLinkTemplate = $('#hashtag-link-template').clone(true);
+
+                    hashtagLinkTemplate.removeAttr('id');
+                    hashtagLinkTemplate.attr('href',$.MAL.hashtagUrl(ret[i]));
+                    hashtagLinkTemplate.text('#'+ret[i]);
+
+                    $li.append(hashtagLinkTemplate);
+                    if ($.Options.getFilterLangSimulateOpt()) {
+                        if (typeof(langFilterData) !== 'undefined') {
+                            $li.append(' <span class="langFilterSimData"><em>'+((langFilterData['pass']) ? polyglot.t('passed') : polyglot.t('blocked'))+'</em>: '+langFilterData['prob'][0].toString()+'</span>');
+                        } else {
+                            $li.append(' <span class="langFilterSimData"><em>'+polyglot.t('not analyzed')+'</em></span>');
+                        }
+                    }
+
+                    $('.toptrends-list').append($li);
+                }
+            }
+        }, {},
+        function(args, ret) {
+            console.log('Error with gettrendinghashtags. Older twister daemon?');
+        }, {}
+    );
+};
+
 //***********************************************
 //******************* INIT **************
 //***********************************************

js/interface_localization.js

@@ -3125,25 +3125,25 @@ if(preferredLanguage == "tr"){
       "hour(s)": "saat",
       "second(s)": "saniye",
       "only numbers are allowed!": "sadece rakam girilebilir!",
-      "Language filtering": "Language filtering",
-      "By blacklist": "By blacklist",
-      "By whitelist": "By whitelist",
-      "Comma separated ISO 639-3 language codes": "Comma separated ISO 639-3 language codes",
-      "Accuracy": "Accuracy",
-      "Simulation mode": "Simulation mode",
-      "This post is treated by language filter": "This post is %{treated} by language filter.",
-      "blocked": "blocked",
-      "passed": "passed",
-      "not analyzed": "not analyzed",
-      "Reason: this": "Reason: %{this}",
-      "this doesnt contain that": "%{this} does not contain %{that}",
-      "blacklist": "blacklist",
-      "whitelist": "whitelist",
-      "language of this": "language of this",
-      "its undefined language": "it is undefined language",
-      "its this, blacklisted": "it is %{this}, blacklisted",
-      "its this, whitelisted": "it is %{this}, whitelisted",
-      "Most possible language: this": "Most possible language: %{this}",
+      "Language filtering": "Dile göre süzme",
+      "By blacklist": "Kara liste ile",
+      "By whitelist": "Beyaz liste ile",
+      "Comma separated ISO 639-3 language codes": "Virgülle ayrılmış ISO 639-3 dil kodları",
+      "Accuracy": "Hassasiyet",
+      "Simulation mode": "Benzetim yap",
+      "This post is treated by language filter": "Bu gönderi dil süzgeci tarafından %{treated}.",
+      "blocked": "engellendi",
+      "passed": "kabul edildi",
+      "not analyzed": "incelenmedi",
+      "Reason: this": "Sebep: %{this}",
+      "this doesnt contain that": "%{that} %{this} tarafından kapsanmıyor",
+      "blacklist": "beyaz liste",
+      "whitelist": "kara liste",
+      "language of this": "dili",
+      "its undefined language": "belirsiz dil",
+      "its this, blacklisted": "%{this}, engellendi",
+      "its this, whitelisted": "%{this}, kabul edildi",
+      "Most possible language: this": "Muhtemel dil: %{this}",
       "Show with every user name": "Tüm kullanıcı adlarının yanında göster",
       "Show at profile modal only": "Sadece profilinde göster",
       "Show if a user follows me": "Bir kullanıcının beni takip edip etmediğini göster",

js/mobile_abstract.js

@@ -488,7 +488,7 @@ var MAL = function()
 
 jQuery.MAL = new MAL;
 
-function filterLangPost(post) {
+function filterLang(string) {
     if ($.Options.getFilterLangOpt() !== 'disable' && $.Options.getFilterLangListOpt().length > 0) {
         var langFilterAccuracy = $.Options.getFilterLangAccuracyOpt();
         var langFilterList = $.Options.getFilterLangListOpt();
@@ -497,14 +497,18 @@ function filterLangPost(post) {
         var langFilterPass = ($.Options.getFilterLangOpt() === 'whitelist') ? false : true;
         var langFilterReason = polyglot.t('this doesnt contain that', {'this': polyglot.t($.Options.getFilterLangOpt()), 'that': polyglot.t('language of this')});
 
-        if (typeof(post['userpost']['rt']) !== 'undefined') {
-            langFilterSubj = post['userpost']['rt']['msg'];
-        } else {
-            langFilterSubj = post['userpost']['msg'];
-        }
-        // we cut out any mentions, links and # symbols from hashtags and clear spaces before detection attempts:
-        langFilterSubj = langFilterSubj.replace(/\@\S\w*|https?:\/\/\S*|\#/g, '').replace(/\s+/g, ' ').trim();
-        langFilterProb = franc.all(langFilterSubj, {'minLength': 4}); // FIXME minLength may become configurable option at some time
+        // before detection attempts we cut out any mentions and links, and replace _ with space
+        langFilterSubj = string.replace(/@\S\w*|https?:\/\/\S*/g, '').replace(/_+/g, ' ')
+        // replace zero-width word boundaries, such as between letters from different alphabets [or other symbols], with spaces
+              // FIXME not so good idea because 'Za pomocą białej listy' may turn into 'Za pomoc ą bia ł ej listy' for e.g.
+              // FIXME but first one was recognized as 'hrv' and second as 'pol' and you know it's 'pol' actually
+            .replace(/\b/g, ' ')
+        // cut out some more symbols
+            .replace(/[#\[\]\(\)\{\}\-\+\=\^\:\;\\\/]/g, '')
+        // clear unwanted spaces
+            .replace(/\s+/g, ' ').trim();
+
+        langFilterProb = franc.all(langFilterSubj, {'minLength': 2}); // FIXME minLength may become configurable option at some time
         for (var i = 0; i < langFilterProb.length; i++) {
             if (langFilterProb[i][1] > langFilterAccuracy) {
                 if (langFilterProb[i][0] === 'und') { // e.g. digits-only string will be detected as undefined and thereby will be allowed
@@ -524,14 +528,9 @@ function filterLangPost(post) {
                 }
             }
         }
-        post['langFilter'] = {};
-        post['langFilter']['subj'] = langFilterSubj;
-        post['langFilter']['prob'] = langFilterProb;
-        post['langFilter']['pass'] = langFilterPass;
-        post['langFilter']['reason'] = langFilterReason;
 
         //console.log('langFilter | status: '+((langFilterPass === true) ? polyglot.t('passed') : polyglot.t('blocked'))+' | reason: '+langFilterReason+' | subject: \''+langFilterSubj+'\'');
-        return ($.Options.getFilterLangSimulateOpt()) ? true : langFilterPass;
+        return {'subj': langFilterSubj, 'prob': langFilterProb, 'pass': langFilterPass, 'reason': langFilterReason};
     }
 }
 

js/options.js

@@ -477,6 +477,36 @@ var TwisterOptions = function()
         });
     }
 
+    this.getFilterLangForPostboardOpt = function () {
+        return $.Options.getOption('filterLangForPostboard', true);
+    }
+
+    this.setFilterLangForPostboardOpt = function () {
+        $('#filterLangForPostboard').prop('checked', $.Options.getFilterLangForPostboardOpt());
+
+        $('#filterLangForPostboard').on('click', function () {$.Options.setOption(this.id, this.checked);});
+    }
+
+    this.getFilterLangForSearchingOpt = function () {
+        return $.Options.getOption('filterLangForSearching', true);
+    }
+
+    this.setFilterLangForSearchingOpt = function () {
+        $('#filterLangForSearching').prop('checked', $.Options.getFilterLangForSearchingOpt());
+
+        $('#filterLangForSearching').on('click', function () {$.Options.setOption(this.id, this.checked);});
+    }
+
+    this.getFilterLangForTopTrendsOpt = function () {
+        return $.Options.getOption('filterLangForTopTrends', true);
+    }
+
+    this.setFilterLangForTopTrendsOpt = function () {
+        $('#filterLangForTopTrends').prop('checked', $.Options.getFilterLangForTopTrendsOpt());
+
+        $('#filterLangForTopTrends').on('click', function () {$.Options.setOption(this.id, this.checked);});
+    }
+
     this.getFilterLangSimulateOpt = function () {
         return $.Options.getOption('filterLangSimulate', false);
     }
@@ -538,6 +568,9 @@ var TwisterOptions = function()
         this.setFilterLangOpt();
         this.setFilterLangListOpt();
         this.setFilterLangAccuracyOpt();
+        this.setFilterLangForPostboardOpt();
+        this.setFilterLangForSearchingOpt();
+        this.setFilterLangForTopTrendsOpt();
         this.setFilterLangSimulateOpt();
         this.setIsFollowingMeOpt();
         this.setDMCopySelfOpt();

js/twister_actions.js

@@ -340,6 +340,22 @@ function processHashtag(postboard, hashtag, data) {
             var key = userpost["n"] + ";" + userpost["time"];
             if( !(key in _hashtagProcessedMap) ) {
                 _hashtagProcessedMap[key] = true;
+
+                if ($.Options.getFilterLangForSearchingOpt()) {
+                    if (typeof(userpost['rt']) !== 'undefined') {
+                        var msg = userpost['rt']['msg'];
+                    } else {
+                        var msg = userpost['msg'];
+                    }
+                    langFilterData = filterLang(msg);
+                    if ($.Options.getFilterLangSimulateOpt()) {
+                        data[i]['langFilter'] = langFilterData;
+                    } else {
+                        if (!langFilterData['pass'])
+                            continue;
+                    }
+                }
+
                 _hashtagPendingPosts.push(data[i]);
                 _hashtagPendingPostsUpdated++;
             }
@@ -363,8 +379,11 @@ function processHashtag(postboard, hashtag, data) {
 
 function displayHashtagPending(postboard) {
     for( var i = 0; i < _hashtagPendingPosts.length; i++ ) {
-        var streamPost = postToElem(_hashtagPendingPosts[i], "original");
-        var timePost = _hashtagPendingPosts[i]["userpost"]["time"];
+        var post = _hashtagPendingPosts[i];
+        //console.log(post);
+        var streamPost = postToElem(post, "original");
+        var timePost = post["userpost"]["time"];
+        streamPost.attr("data-time",timePost);
 
         var streamItems = postboard.children();
         if( streamItems.length == 0) {

js/twister_formatpost.js

@@ -119,8 +119,19 @@ function postToElem( post, kind, promoted ) {
         retweetedByElem.text('@'+retweeted_by);
     }
 
-    if (typeof(promoted) !== 'undefined' && promoted)
+    if (typeof(promoted) !== 'undefined' && promoted) {
         elem.find('.post-propagate').remove();
+    } else {
+        if ($.Options.getFilterLangSimulateOpt()) {
+            // FIXME it's must be stuff from template actually
+            if (typeof(post['langFilter']) !== 'undefined') {
+                elem.append('<div class="langFilterSimData">'+polyglot.t('This post is treated by language filter', {'treated': '<em>'+((post['langFilter']['pass']) ? polyglot.t('passed') : polyglot.t('blocked'))+'</em>'})+'</div>');
+                elem.append('<div class="langFilterSimData">'+polyglot.t('Reason: this', {'this': '<em>'+post['langFilter']['reason']+'</em>'})+'  //  '+polyglot.t('Most possible language: this', {'this': '<em>'+post['langFilter']['prob'][0].toString()+'</em>'})+'</div>');
+            } else {
+                elem.append('<div class="langFilterSimData">'+polyglot.t('This post is treated by language filter', {'treated': '<em>'+polyglot.t('not analyzed')+'</em>'})+'</div>');
+            }
+        }
+    }
 
     return elem;
 }

js/twister_timeline.js

@@ -235,15 +235,6 @@ function showPosts(req, posts)
             }
 
             if( streamPostAppended ) {
-                if ($.Options.getFilterLangSimulateOpt()) {
-                    // FIXME it's should be stuff from template actually
-                    if (typeof(post['langFilter']) !== 'undefined') {
-                        streamPost.append('<div class="langFilterSimData">'+polyglot.t('This post is treated by language filter', {'treated': '<em>'+((post['langFilter']['pass'] === true) ? polyglot.t('passed') : polyglot.t('blocked'))+'</em>'})+'</div>');
-                        streamPost.append('<div class="langFilterSimData">'+polyglot.t('Reason: this', {'this': '<em>'+post['langFilter']['reason']+'</em>'})+'  //  '+polyglot.t('Most possible language: this', {'this': '<em>'+post['langFilter']['prob'][0].toString()+'</em>'})+'</div>');
-                    } else {
-                        streamPost.append('<div class="langFilterSimData">'+polyglot.t('This post is treated by language filter', {'treated': '<em>'+polyglot.t('not analyzed')+'</em>'})+'</div>');
-                    }
-                }
                 streamPost.show();
             }
             req.reportProcessedPost(post["userpost"]["n"],post["userpost"]["k"], streamPostAppended);
@@ -371,14 +362,21 @@ function willBeHidden(post){
         return false;
 
     if (typeof(post['userpost']['rt']) !== 'undefined') {
+        // hope it is not too egocentric to overcome HideCloseRTsOpt this way
+        if (post['userpost']['rt']['n'] === defaultScreenName)
+            return false;
+
         if ($.Options.getHideCloseRTsOpt() != 'disable' &&
             followingUsers.indexOf(post['userpost']['rt']['n']) > -1 &&
             parseInt(post['userpost']['time']) - parseInt(post['userpost']['rt']['time']) < $.Options.getHideCloseRTsHourOpt() * 3600)
         {
             return true;
         }
+
+        var msg = post['userpost']['rt']['msg'];
     } else {
         var msg = post['userpost']['msg'];
+
         if ($.Options.getHideRepliesOpt() !== 'disable' &&
             /^\@/.test(msg) &&
             !(new RegExp('@' + defaultScreenName + '( |,|;|\\.|:|\\/|\\?|\\!|\\\\|\'|"|\\n|\\t|$)').test(msg)))
@@ -392,11 +390,14 @@ function willBeHidden(post){
         }
     }
 
-    if (filterLangPost(post) === false) {
+    if ($.Options.getFilterLangForPostboardOpt()) {
+        post['langFilter'] = filterLang(msg);
+        if (!post['langFilter']['pass'] && !$.Options.getFilterLangSimulateOpt()) {
             // TODO maybe we need a counter of posts blocked by language filter and even caching of them and button to show?
             //console.log('post by @'+post['userpost']['n']+' was hidden because it didn\'t passed by language filter:');
             return true;
         }
+    }
 
     return false;
 }

options.html

@@ -240,10 +240,11 @@
                 </p>
               </form>
             </div>
+          </div>
+          <div class="module">
+            <p class="label label-h">Language filtering</p>
             <div class="container">
               <form action="" id="filterLangOpt">
-                <p class="label">Language filtering</p>
-                <div class="container">
                 <p>
                   <select name="" id="filterLang">
                     <option value="disable">Disable</option>
@@ -256,8 +257,13 @@
                   <p><input type="text" id="filterLangList" class="container" /></p>
                   <p class="label">Accuracy</p>
                   <p><input type="range" id="filterLangAccuracy" class="container" min="0" max="1" step="0.005"><span id="filterLangAccuracyVal" class="container"></span></p>
-                    <p><span class="label">Simulation mode</span> <input type="checkbox" id="filterLangSimulate" /></p>
+                  <p><span class="label">Scope of using</span></p>
+                  <div class="container">
+                    <p><span class="label">Postboard</span> <input type="checkbox" id="filterLangForPostboard" /></p>
+                    <p><span class="label">Searching</span> <input type="checkbox" id="filterLangForSearching" /></p>
+                    <p><span class="label">Top Trends</span> <input type="checkbox" id="filterLangForTopTrends" /></p>
                   </div>
+                  <p><span class="label">Simulation mode</span> <input type="checkbox" id="filterLangSimulate" /></p>
                 </div>
               </form>
             </div>