postboard lang filtering, its options, tuning of shining

css/style.css

@@ -1898,8 +1898,7 @@ ol.toptrends-list {
     box-shadow: 0 0 10px rgba(0, 0, 0, .3 );
 }
 
-.volValue {
-    vertical-align: top;
+.volValue, #filterLangAccuracyVal {
     font: 12px "Open Sans", sans-serif;
 }
 
@@ -1915,6 +1914,29 @@ ol.toptrends-list {
     -ms-transition: height 1s linear;
 }
 
+.options input[type='checkbox'] {
+    cursor: pointer;
+    vertical-align: sub;
+}
+
+.options input[type='range'] {
+    cursor: pointer;
+    vertical-align: -16px;
+}
+
+.options #filterLangList {
+    width: 80%;
+}
+
+.langFilterSimData {
+    font: 10px "Open Sans", sans-serif;
+    text-align: center;
+}
+
+.langFilterSimData em {
+    color: rgba( 227, 79, 66, .5 );
+}
+
 /*************************************
 *********** FOLLOWING PAGE ***********
 **************************************/

following.html

@@ -11,6 +11,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>

home.html

@@ -11,6 +11,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>

js/mobile_abstract.js

@@ -488,3 +488,51 @@ var MAL = function()
 
 jQuery.MAL = new MAL;
 
+function filterLangPost(post) {
+    if ($.Options.getFilterLangOpt() !== 'disable' && $.Options.getFilterLangListOpt().length > 0) {
+        var langFilterAccuracy = $.Options.getFilterLangAccuracyOpt();
+        var langFilterList = $.Options.getFilterLangListOpt();
+        var langFilterSubj = '';
+        var langFilterProb = [];
+        var langFilterPass = ($.Options.getFilterLangOpt() === 'whitelist') ? false : true;
+        var langFilterReason = 'language isn\'t in '+$.Options.getFilterLangOpt();
+
+        if (typeof(post['userpost']['rt']) !== 'undefined') {
+            langFilterSubj = post['userpost']['rt']['msg'].replace(/\@\S\w*|https?:\/\/\S*|\#/g, '').replace(/\s+/g, ' ');
+        } else {
+            langFilterSubj = post['userpost']['msg'].replace(/\@\S\w*|https?:\/\/\S*|\#/g, '').replace(/\s+/g, ' ');
+        }
+        langFilterProb = franc.all(langFilterSubj, {'minLength': 5}); // FIXME minLength may be optional sometimes
+        for (var i = 0; i < langFilterProb.length; i++) {
+            if (langFilterProb[i][1] > langFilterAccuracy) {
+                if (langFilterProb[i][0] === 'und') { // FIXME maybe there's should be an option for whitelist mode to allow this
+                    langFilterPass = true;
+                    langFilterReason = 'it\'s undefined language';
+                    break;
+                } else if (langFilterProb[i][0] === 'cmn') {
+                    langFilterPass = true;
+                    langFilterReason = 'it\'s common language';
+                    break;
+                } else if (langFilterList.indexOf(langFilterProb[i][0]) > -1) {
+                    if ($.Options.getFilterLangOpt() === 'whitelist') {
+                        langFilterPass = true;
+                        langFilterReason = 'it\'s '+langFilterProb[i][0]+', whitelisted';
+                        break;
+                    } else {
+                        langFilterPass = false;
+                        langFilterReason = 'it\'s '+langFilterProb[i][0]+', blacklisted';
+                        break;
+                    }
+                }
+            }
+        }
+        post['langFilter'] = {};
+        post['langFilter']['subj'] = langFilterSubj;
+        post['langFilter']['prob'] = langFilterProb;
+        post['langFilter']['pass'] = langFilterPass;
+        post['langFilter']['reason'] = langFilterReason;
+
+        return ($.Options.getFilterLangSimulateOpt()) ? true : langFilterPass;
+    }
+}
+

js/options.js

@@ -434,6 +434,59 @@ var TwisterOptions = function()
         return $.Options.getOption('isFollowingMe');
     };
 
+    this.getFilterLangOpt = function() {
+        return $.Options.getOption('filterLang','disable');
+    }
+
+    this.setFilterLangOpt = function () {
+        function filterLangListCont() {
+            if ( $.Options.getFilterLangOpt() !== 'disable' ) {
+                $('#filterLangListCont')[0].style.display= 'block';
+            } else {
+                $('#filterLangListCont')[0].style.display= 'none';
+            }
+        }
+        $('#filterLang').val(this.getFilterLangOpt());
+        filterLangListCont();
+        $('#filterLang').on('change', function() {
+            $.Options.setOption(this.id, this.value);
+            filterLangListCont();
+        });
+    }
+
+    this.getFilterLangListOpt = function () {
+        return $.Options.getOption('filterLangList', '').split(/\s*,\s*/);
+    }
+
+    this.setFilterLangListOpt = function () {
+        $('#filterLangList').val(this.getFilterLangListOpt());
+
+        $('#filterLangList').on('keyup', function () {$.Options.setOption(this.id, this.value);});
+    }
+
+    this.getFilterLangAccuracyOpt = function () {
+        return parseFloat($.Options.getOption('filterLangAccuracy', '0.63'));
+    }
+
+    this.setFilterLangAccuracyOpt = function () {
+        $('#filterLangAccuracy').val(this.getFilterLangAccuracyOpt());
+        $('#filterLangAccuracyVal').text(this.getFilterLangAccuracyOpt());
+        $('#filterLangAccuracy').on('change', function () {
+            $.Options.setOption(this.id, this.value);
+            $('#filterLangAccuracyVal').text(this.value);
+        });
+    }
+
+    this.getFilterLangSimulateOpt = function () {
+        return $.Options.getOption('filterLangSimulate', false);
+    }
+
+    this.setFilterLangSimulateOpt = function () {
+        $('#filterLangSimulate').prop('checked', $.Options.getFilterLangSimulateOpt());
+
+        $('#filterLangSimulate').on('click', function () {$.Options.setOption(this.id, this.checked);});
+    }
+
     this.setIsFollowingMeOpt = function () {
         $('#isFollowingMe')[0].value = this.getIsFollowingMeOpt();
 
@@ -482,6 +535,10 @@ var TwisterOptions = function()
         this.setHideRepliesOpt();
         this.setHideCloseRTsHourOpt();
         this.setHideCloseRTsOpt();
+        this.setFilterLangOpt();
+        this.setFilterLangListOpt();
+        this.setFilterLangAccuracyOpt();
+        this.setFilterLangSimulateOpt();
         this.setIsFollowingMeOpt();
         this.setDMCopySelfOpt();
     }

js/twister_actions.js

@@ -389,4 +389,3 @@ function displayHashtagPending(postboard) {
     $.MAL.postboardLoaded();
     _hashtagPendingPosts = [];
 }
-

js/twister_timeline.js

@@ -112,23 +112,24 @@ function requestObj(users, mode, count, getspam)
     // doneReportProcessing is called after an getposts response is processed
     // mode changing may require a new request (to fill gaps)
     this.doneReportProcessing = function(receivedCount) {
-        if (receivedCount === this.count) {
+        if (receivedCount >= this.count) {
             this.mode = 'done';
         } else {
             if (this.mode === 'latest' || this.mode === 'latestFirstTime') {
                 this.mode = 'fillgap';
             } else if (this.mode === 'fillgap') {
                 this.mode = 'older';
-            } else if (this.mode === 'older')
-                this.mode = 'done';
+            }
         }
-        //console.log('we got '+receivedCount+' posts from requested '+this.count+', status of processing is "'+this.mode+'"');
+        //console.log('we got '+receivedCount+' posts from requested '+this.count+', status of processing: '+this.mode);
     }
 }
 
 // json rpc with requestObj as parameter
 function requestGetposts(req)
 {
+    //console.log('requestGetposts');
+    //console.log(req);
     var r = req.getRequest();
     if( !req.getspam ) {
         if( r.length ) {
@@ -148,9 +149,12 @@ function requestGetposts(req)
 // request if needed
 function processReceivedPosts(req, posts)
 {
+    //console.log('processReceivedPosts:');
+    //console.log(posts);
     //hiding posts can cause empty postboard, so we have to track the count...
     for( var i = 0; i < posts.length; i++ ) {
         if (willBeHidden(posts[i])) {
+            req.reportProcessedPost(posts[i]["userpost"]["n"],posts[i]["userpost"]["k"], true)
             posts.splice(i, 1);
             i--;
         }
@@ -168,7 +172,7 @@ function processReceivedPosts(req, posts)
         req.count -= posts.length;
         if (req.count > 0) {
             //console.log('we are requesting '+req.count+' more posts...');
-            requestGetposts(req);
+            setTimeout((function (){requestGetposts(this)}).bind(req), 1000);
         } else {
             timelineLoaded = true;
             $.MAL.postboardLoaded();
@@ -179,12 +183,15 @@ function processReceivedPosts(req, posts)
 
 function showPosts(req, posts)
 {
+    //console.log('showPosts:');
+    //console.log(req);
+    //console.log(posts);
     var streamItemsParent = $.MAL.getStreamPostsParent();
 
     for( var i = 0; i < posts.length; i++ ) {
         if ( req.users.indexOf(posts[i]['userpost']['n']) > -1 ) {
             var post = posts[i];
-
+            //console.log(post);
             var streamPost = postToElem(post, "original", req.getspam);
             var timePost = post["userpost"]["time"];
             streamPost.attr("data-time",timePost);
@@ -228,6 +235,15 @@ function showPosts(req, posts)
             }
 
             if( streamPostAppended ) {
+                if ($.Options.getFilterLangSimulateOpt()) {
+                    // FIXME it's should be stuff from template actually
+                    if (typeof(post['langFilter']) !== 'undefined') {
+                        streamPost.append('<div class="langFilterSimData">This post is <em>'+((post['langFilter']['pass'] === true) ? 'passed' : 'blocked')+'</em> by language filter.</div>');
+                        streamPost.append('<div class="langFilterSimData">Reason: <em>'+post['langFilter']['reason']+'</em> // Most possible language: <em>'+post['langFilter']['prob'][0].toString()+'</em></div>');
+                    } else {
+                        streamPost.append('<div class="langFilterSimData">This post is not filtered by language.</div>');
+                    }
+                }
                 streamPost.show();
             }
             req.reportProcessedPost(post["userpost"]["n"],post["userpost"]["k"], streamPostAppended);
@@ -297,6 +313,8 @@ function processLastHave(userHaves)
 
     // now do a getposts to confirm the number of new haves with are effectively new public posts
     if( newPostsLocal ) {
+        //console.log('processLastHave(): requesting '+newPostsLocal);
+        //console.log(reqConfirmNewPosts);
         twisterRpc("getposts", [newPostsLocal, reqConfirmNewPosts],
                function(expected, posts) {processNewPostsConfirmation(expected, posts);}, newPostsLocal,
                function(req, ret) {console.log("ajax error:" + ret);}, null);
@@ -378,5 +396,10 @@ function willBeHidden(post){
             return true;
     }
 
+    if (filterLangPost(post) === false) {
+        hidden = true;
+        console.log('post by @'+post['userpost']['n']+' was hidden because it didn\'t passed by language filter');
+    }
+
     return hidden;
 }

login.html

@@ -10,6 +10,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>

network.html

@@ -10,6 +10,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>

options.html

@@ -10,6 +10,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>
@@ -239,6 +240,27 @@
                 </p>
               </form>
             </div>
+            <div class="container">
+              <form action="" id="filterLangOpt">
+                <p class="label">Language filtering</p>
+                <div class="container">
+                  <p>
+                  <select name="" id="filterLang">
+                    <option value="disable">Disable</option>
+                    <option value="blacklist">By blacklist</option>
+                    <option value="whitelist">By whitelist</option>
+                  </select>
+                  </p>
+                  <div id="filterLangListCont" class="container">
+                    <p class="label">Comma separated ISO 639-3 language codes</p>
+                    <p><input type="text" id="filterLangList" class="container" /></p>
+                    <p class="label">Accuracy</p>
+                    <p><input type="range" id="filterLangAccuracy" class="container" min="0" max="1" step="0.01"><span id="filterLangAccuracyVal" class="container"></span></p>
+                    <p><span class="label">Simulation mode</span> <input type="checkbox" id="filterLangSimulate" /></p>
+                  </div>
+                </div>
+              </form>
+            </div>
           </div>
           <div class="module">
             <p class="label label-h"> Posts display </p>

profile-edit.html

@@ -10,6 +10,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js"></script>
     <script src="js/twister_io.js"></script>

theme_calm/css/style.css

@@ -2327,7 +2327,7 @@ textarea.splited-post {
 
 .options .module select {
     padding: 0px 30px 0px 10px;
-    text-align: center;
+    text-align: left;
     background: #fff url(../img/form-arrow-down-black.png) no-repeat right center;
     border: 1px solid #ccc;
     -webkit-border-radius: 4px;
@@ -2343,8 +2343,7 @@ textarea.splited-post {
 /* end */
 }
 
-.volValue {
-    vertical-align: top;
+.volValue, #filterLangAccuracyVal {
     font: 12px "Open Sans", sans-serif;
 }
 
@@ -2360,6 +2359,30 @@ textarea.splited-post {
     -ms-transition: height 1s linear;
 }
 
+.options input[type='checkbox'] {
+    cursor: pointer;
+    vertical-align: sub;
+}
+
+.options input[type='range'] {
+    cursor: pointer;
+    vertical-align: -16px;
+}
+
+.options #filterLangList {
+    width: 80%;
+    text-align: left;
+}
+
+.langFilterSimData {
+    font: 10px "Open Sans", sans-serif;
+    text-align: center;
+}
+
+.langFilterSimData em {
+    color: #768fce;
+}
+
 /*************************************
 *********** FOLLOWING PAGE ***********
 **************************************/

theme_nin/css/style.css

@@ -1740,6 +1740,11 @@ button.disabled:hover, .mini-profile-actions span.disabled:hover, a.button.disab
 {
     margin: 5px 0px 5px 10px;
 }
+
+.options button, .options input, .options select {
+  font-size: 13px;
+}
+
 /* line 9, ../sass/_tabs.sass */
 .options input[type=radio] {
   display: none !important;
@@ -1804,6 +1809,30 @@ button.disabled:hover, .mini-profile-actions span.disabled:hover, a.button.disab
   color: #66686B;
 }
 
+.options input[type='checkbox'] {
+    cursor: pointer;
+    vertical-align: sub;
+}
+
+.options input[type='range'] {
+    cursor: pointer;
+    vertical-align: -16px;
+}
+
+.options #filterLangList {
+    width: 100%;
+}
+
+.langFilterSimData {
+    font: 10px "Open Sans", sans-serif;
+    text-align: center;
+}
+
+.langFilterSimData em {
+    color: #B4C669;
+}
+
+
 /* line 19, ../sass/style.sass */
 .promoted-posts-only {
   margin: 0 0;

tmobile.html

@@ -28,6 +28,7 @@
     <script src="js/jquery.jsonrpcclient.js"></script>
     <script src="js/jquery.storageapi.js"></script>
     <script src="js/options.js?vr=10"></script>
+    <script src="js/franc.js"></script>
     <script src="js/notify.js"></script>
     <script src="js/mobile_abstract.js?vr=10"></script>
     <script src="js/twister_io.js?vr=10"></script>