term_test.py

 # -*- coding: UTF-8 -*-

import unittest
import os
import sys
from term import StopList, StoplistFileError
	
class StopListTestCase(unittest.TestCase):
    
    def create_words_file(self, filename, words):
        words_file = open(filename, 'w')
        if not words_file:
            raise Exception("Unable to create stop file, " + filename)
        for word in words:
            words_file.write(word.encode('utf-8') + u"\n".encode('utf-8'))
        words_file.close()
        
    def setUp(self):
        self.words_filename = 'words.txt'
        self.words = [u'a', u'an', u'the', u'any', u'is', u'from']
        self.create_words_file(self.words_filename, self.words)

        self.addl_words_filename = 'addl_words.txt'
        self.addl_words =  [u'of', u'with', u'to', u'for']
        self.create_words_file(self.addl_words_filename, self.addl_words)

        self.non_english_words_filename = 'non_english_words.txt'
        self.non_english_words = [u'ça', u'était', u'à', u'über']
        self.create_words_file(self.non_english_words_filename, self.non_english_words)

        self.comma_in_word_filename = 'words_comma.txt'
        self.comma_in_word_lines = [u'a,an']
        self.create_words_file(self.comma_in_word_filename, self.comma_in_word_lines)

        self.space_in_word_filename = 'words_space.txt'
        self.space_in_word_lines = [u"ça était"]
        self.create_words_file(self.space_in_word_filename, self.space_in_word_lines)

        self.empty_stoplist = StopList()
        self.stoplist = StopList(self.words_filename)
        
    def tearDown(self):
        os.unlink(self.words_filename)
        os.unlink(self.addl_words_filename)
        os.unlink(self.non_english_words_filename)
        os.unlink(self.comma_in_word_filename)
        os.unlink(self.space_in_word_filename)
        
    def test_wordset(self):
        assert self.empty_stoplist.wordset == set([])
        assert len(self.stoplist.wordset) == len(self.words)
        assert u'an' in self.stoplist.wordset
        assert u'obama' not in self.stoplist.wordset
        
    def test_wordset_not_setable(self):
        try :
            self.stoplist.wordset = set([])
        except AttributeError:
            pass
        except:
            self.fail("Expected AttributeError got:", sys.exc_info()[0])
        else:
            self.fail("Property wordset should not be setable")
        
    def test_iteration(self):
        words = self.words[:]
        for word in self.stoplist:
            assert word in words
            words.remove(word)
        assert len(words) == 0

    def test_in_operator(self):
        assert u'an' in self.stoplist
        assert u'AN' in self.stoplist
        assert u'obama' not in self.stoplist
        
    def test_load_file(self):
        self.stoplist.load_file(self.addl_words_filename)
        assert len(self.stoplist.wordset) == \
               len(self.words) + len(self.addl_words)
        for word in self.words + self.addl_words:
            assert word in self.stoplist
            
    def test_load_nonexistent_file(self):
        self.assertRaises(IOError, 
                          self.stoplist.load_file, 
                          'this_file_does_not_exist.txt')

    def test_unicode_support(self):
        self.stoplist.load_file(self.non_english_words_filename)
        for word in self.non_english_words:
            assert word in self.stoplist
    
    def test_malformed_file(self):
        self.assertRaises(StoplistFileError, 
                          self.stoplist.load_file, 
                          self.comma_in_word_filename)
        self.assertRaises(StoplistFileError, 
                          self.empty_stoplist.load_file, 
                          self.space_in_word_filename)
            
    def test_remove_stopwords(self):
        clark_quote = 'Any sufficiently advanced technology is indistinguishable from magic'
        quote_words = str.split(clark_quote)
        quote_words_sans_stopwords = [u'sufficiently', u'advanced', u'technology', 
                                      u'indistinguishable', u'magic']
        assert quote_words_sans_stopwords == self.stoplist.remove_stopwords(quote_words)
        assert None == self.stoplist.remove_stopwords(None)
        assert [] == self.stoplist.remove_stopwords([])