Ticket #370: Spam.cobra

use System.Net
use System.Web
use System.IO
use System.Text

use CherryCobbler.Common

#Cobra Port of: https://github.com/rrenaud/Gibberish-Detector
class GibberishTrainer
    var _accepted_chars = "abcdefghijklmnopqrstuvwxyz"

    var _charIndex as Dictionary<of char,int>

    cue init
        _charIndex = Dictionary<of char,int>()
        i =0
        for c in _accepted_chars
            _charIndex[c] = i
            i +=1

    def normalize(line as String) as String
        """Return the subset of acceptable characters.  Ignores punctuation, infrequent symbols, etc..."""
        sb = StringBuilder()
        for c in line.toLower
            if _charIndex.containsKey(c)
                sb.append(c)
        return sb.toString
    
    def ngram(n as int, toParse as String) as String*
        """Returns all n grams from toParse after normalizing"""
        filtered = .normalize(toParse)
        for start in filtered.length - n +1
            yield String.join(filtered[start:start+n]) to !

    def getFile as String
        url = "http://norvig.com/big.txt"
        try
            wc = WebClient()
            return wc.downloadString(url) to !
        catch e as Exception
            print "getFile error: [e.toString]"
            return ""
    
    def train
        charsetSize = _accepted_chars.length

        #Assume we have seen 10 of each character pair.  This is a smoothing, to 
        #allow for unseen sequences to not automatically be 0 probability.
        counts = for i in charsetSize get for k in charsetSize get 10.0

        #count transition from big text file, taken from http://norvig.com/spell-correct.html
        text = .getFile
        sr = StringReader(text)
        line = sr.readLine
        post while line is not nil
            for gram in .ngram(2,line to !)
                counts[_charIndex[gram[0]]][_charIndex[gram[1]]] +=1
            
            line = sr.readLine
        
        # Normalize the counts so they become log probabilities.  
        # We use log probabilities rather than straight probabilities to avoid numeric underflow issues with long texts.
        # This contains a justification:
        # http://squarecog.wordpress.com/2009/01/10/dealing-with-underflow-in-joint-probability-calculations/
        for r in counts
            sum = 0.0
            for c in r
                sum+=c