Multinomial and Bernoulli Naive Bayes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

import numpy as np
from collections import Counter
from typing import List, Dict, Tuple
 
class BagOfWordsVectorizer:
    """
    A complete implementation of Bag-of-Words vectorization.
    
    This class transforms text documents into numerical feature vectors
    using word frequency counts.
    """
    
    def __init__(self, min_df: int = 1, max_df: float = 1.0):
        """
        Initialize the vectorizer.
        
        Args:
            min_df: Minimum document frequency for a word to be included
            max_df: Maximum document frequency (as fraction) for inclusion
        """
        self.vocabulary_: Dict[str, int] = {}
        self.idf_: np.ndarray = None
        self.min_df = min_df
        self.max_df = max_df
        
    def _tokenize(self, document: str) -> List[str]:
        """
        Simple whitespace tokenization with lowercasing.
        In production, you'd use more sophisticated tokenization.
        """
        # Convert to lowercase and split on whitespace
        tokens = document.lower().split()
        # Remove punctuation (simplified)
        tokens = [token.strip('.,!?;:"\'()[]{}') for token in tokens]
        # Remove empty tokens
        return [token for token in tokens if token]
    
    def fit(self, documents: List[str]) -> 'BagOfWordsVectorizer':
        """
        Learn vocabulary from training documents.
        
        Args:
            documents: List of text documents
            
        Returns:
            self for method chaining
        """
        n_docs = len(documents)
        
        # Count document frequency for each word
        doc_freq = Counter()
        for doc in documents:
            tokens = set(self._tokenize(doc))  # Unique tokens per doc
            doc_freq.update(tokens)
        
        # Filter by document frequency thresholds
        max_count = int(self.max_df * n_docs)
        vocabulary = {
            word: idx 
            for idx, (word, count) in enumerate(doc_freq.items())
            if self.min_df <= count <= max_count
        }
        
        self.vocabulary_ = vocabulary
        
        # Compute IDF weights for TF-IDF (optional enhancement)
        self.idf_ = np.array([
            np.log((n_docs + 1) / (doc_freq[word] + 1)) + 1
            for word in vocabulary
        ])
        
        return self
    
    def transform(self, documents: List[str]) -> np.ndarray:
        """
        Transform documents to BoW feature vectors.
        
        Args:
            documents: List of text documents
            
        Returns:
            Document-term matrix of shape (n_docs, n_features)
        """
        n_docs = len(documents)
        n_features = len(self.vocabulary_)
        
        # Initialize sparse-like dense matrix (for simplicity)
        X = np.zeros((n_docs, n_features), dtype=np.float64)
        
        for i, doc in enumerate(documents):
            tokens = self._tokenize(doc)
            token_counts = Counter(tokens)
            
            for token, count in token_counts.items():
                if token in self.vocabulary_:
                    j = self.vocabulary_[token]
                    X[i, j] = count
                    
        return X
    
    def fit_transform(self, documents: List[str]) -> np.ndarray:
        """Fit and transform in one step."""
        return self.fit(documents).transform(documents)
 
 
# Example usage demonstrating the complete pipeline
documents = [
    "The quick brown fox jumps over the lazy dog",
    "The lazy dog sleeps all day",
    "Quick brown foxes are faster than lazy dogs",
    "The dog chased the cat across the yard"
]
 
vectorizer = BagOfWordsVectorizer(min_df=1)
X = vectorizer.fit_transform(documents)
 
print("Vocabulary:")
for word, idx in sorted(vectorizer.vocabulary_.items(), key=lambda x: x[1]):
    print(f"  {word}: index {idx}")
 
print(f"
Document-term matrix shape: {X.shape}")
print(f"
First document vector (non-zero entries):")
for word, idx in vectorizer.vocabulary_.items():
    if X[0, idx] > 0:
        print(f"  {word}: {X[0, idx]}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

import numpy as np
from collections import Counter
from typing import List, Dict
 
class TFIDFVectorizer:
    """
    TF-IDF Vectorizer with multiple normalization options.
    
    Demonstrates the mathematical transformation from raw counts
    to weighted representations that emphasize discriminative words.
    """
    
    def __init__(self, 
                 norm: str = 'l2',
                 use_idf: bool = True,
                 smooth_idf: bool = True,
                 sublinear_tf: bool = False):
        """
        Args:
            norm: Normalization method ('l1', 'l2', or None)
            use_idf: Whether to apply IDF weighting
            smooth_idf: Add 1 to document frequencies (prevents division by zero)
            sublinear_tf: Apply sublinear TF scaling (1 + log(tf))
        """
        self.norm = norm
        self.use_idf = use_idf
        self.smooth_idf = smooth_idf
        self.sublinear_tf = sublinear_tf
        self.vocabulary_: Dict[str, int] = {}
        self.idf_: np.ndarray = None
        
    def _tokenize(self, text: str) -> List[str]:
        """Simple tokenization."""
        return text.lower().split()
    
    def fit(self, documents: List[str]) -> 'TFIDFVectorizer':
        """Learn vocabulary and IDF weights from documents."""
        n_docs = len(documents)
        
        # Build vocabulary and count document frequencies
        doc_freq = Counter()
        word_to_idx = {}
        idx = 0
        
        for doc in documents:
            unique_tokens = set(self._tokenize(doc))
            for token in unique_tokens:
                if token not in word_to_idx:
                    word_to_idx[token] = idx
                    idx += 1
            doc_freq.update(unique_tokens)
        
        self.vocabulary_ = word_to_idx
        n_features = len(word_to_idx)
        
        # Compute IDF weights
        if self.use_idf:
            # idf(t) = log[(N + 1) / (df(t) + 1)] + 1 (smoothed)
            # or idf(t) = log[N / df(t)] + 1 (unsmoothed)
            df = np.ones(n_features)  # Default 1 for smoothing
            for word, count in doc_freq.items():
                df[word_to_idx[word]] = count
                
            if self.smooth_idf:
                self.idf_ = np.log((n_docs + 1) / (df + 1)) + 1
            else:
                self.idf_ = np.log(n_docs / df) + 1
        else:
            self.idf_ = np.ones(n_features)
            
        return self
    
    def transform(self, documents: List[str]) -> np.ndarray:
        """Transform documents to TF-IDF representation."""
        n_docs = len(documents)
        n_features = len(self.vocabulary_)
        
        # Compute term frequencies
        X = np.zeros((n_docs, n_features))
        
        for i, doc in enumerate(documents):
            tokens = self._tokenize(doc)
            counts = Counter(tokens)
            
            for token, count in counts.items():
                if token in self.vocabulary_:
                    j = self.vocabulary_[token]
                    if self.sublinear_tf:
                        # Sublinear scaling: 1 + log(tf) if tf > 0
                        X[i, j] = 1 + np.log(count) if count > 0 else 0
                    else:
                        X[i, j] = count
        
        # Apply IDF weighting
        X = X * self.idf_
        
        # Apply normalization
        if self.norm == 'l2':
            # L2 normalization: divide each row by its L2 norm
            norms = np.sqrt(np.sum(X ** 2, axis=1, keepdims=True))
            norms[norms == 0] = 1  # Prevent division by zero
            X = X / norms
        elif self.norm == 'l1':
            # L1 normalization: divide each row by its L1 norm
            norms = np.sum(np.abs(X), axis=1, keepdims=True)
            norms[norms == 0] = 1
            X = X / norms
            
        return X
 
 
# Demonstrate how TF-IDF distinguishes discriminative words
documents = [
    "the cat sat on the mat",           # Common words dominate
    "the dog sat on the rug",           # Similar structure
    "machine learning is transforming artificial intelligence",
    "deep learning neural networks artificial intelligence"
]
 
vectorizer = TFIDFVectorizer(norm='l2', smooth_idf=True)
X = vectorizer.fit_transform(documents)
 
print("TF-IDF Analysis:")
print("-" * 50)
for word, idx in sorted(vectorizer.vocabulary_.items(), 
                        key=lambda x: vectorizer.idf_[x[1]], 
                        reverse=True)[:10]:
    print(f"  {word:20s} IDF: {vectorizer.idf_[idx]:.4f}")
 
print(f"
Document similarity matrix (cosine similarity):")
similarity = X @ X.T  # Cosine similarity (vectors are L2-normalized)
print(similarity.round(3))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

import numpy as np
from collections import defaultdict
from typing import List, Dict, Tuple
 
class GenerativeTextModel:
    """
    Illustrates the generative model perspective of Naive Bayes.
    
    This class shows how we can:
    1. Estimate P(c) and P(w|c) from training data
    2. Generate synthetic documents from learned distributions
    3. Classify new documents using Bayes' rule
    
    This demonstrates that classification and generation are two
    sides of the same probabilistic model.
    """
    
    def __init__(self, smoothing: float = 1.0):
        self.smoothing = smoothing
        self.class_priors: Dict[str, float] = {}
        self.word_probs: Dict[str, Dict[str, float]] = {}
        self.vocabulary: List[str] = []
        self.classes: List[str] = []
        
    def fit(self, documents: List[str], labels: List[str]):
        """Estimate model parameters from labeled documents."""
        # Count class frequencies
        class_counts = defaultdict(int)
        for label in labels:
            class_counts[label] += 1
            
        n_docs = len(labels)
        self.classes = list(set(labels))
        
        # Estimate class priors: P(c) = count(c) / N
        self.class_priors = {
            c: count / n_docs 
            for c, count in class_counts.items()
        }
        
        # Count word occurrences per class
        word_counts_per_class = defaultdict(lambda: defaultdict(int))
        all_words = set()
        
        for doc, label in zip(documents, labels):
            words = doc.lower().split()
            all_words.update(words)
            for word in words:
                word_counts_per_class[label][word] += 1
                
        self.vocabulary = sorted(all_words)
        vocab_size = len(self.vocabulary)
        
        # Estimate word probabilities: P(w|c)
        # Using Laplace smoothing: (count(w,c) + α) / (Σ count(w',c) + α|V|)
        self.word_probs = {}
        for c in self.classes:
            total_words = sum(word_counts_per_class[c].values())
            self.word_probs[c] = {
                word: (word_counts_per_class[c][word] + self.smoothing) / 
                      (total_words + self.smoothing * vocab_size)
                for word in self.vocabulary
            }
            
        return self
    
    def generate_document(self, class_label: str, length: int = 20) -> str:
        """
        Generate a document from the learned distribution.
        
        This demonstrates the 'generative' aspect of the model.
        We sample words according to P(w|c).
        """
        if class_label not in self.classes:
            raise ValueError(f"Unknown class: {class_label}")
            
        # Sample words according to P(w|class_label)
        words = list(self.word_probs[class_label].keys())
        probs = list(self.word_probs[class_label].values())
        probs = np.array(probs) / np.sum(probs)  # Ensure normalized
        
        sampled_words = np.random.choice(words, size=length, p=probs)
        return ' '.join(sampled_words)
    
    def classify(self, document: str) -> Tuple[str, Dict[str, float]]:
        """
        Classify a document using Bayes' rule.
        
        Returns the predicted class and log-probabilities for all classes.
        """
        words = document.lower().split()
        
        log_posteriors = {}
        for c in self.classes:
            # Start with log prior
            log_prob = np.log(self.class_priors[c])
            
            # Add log-likelihood of each word
            for word in words:
                if word in self.word_probs[c]:
                    log_prob += np.log(self.word_probs[c][word])
                else:
                    # Handle unknown words with minimal probability
                    log_prob += np.log(self.smoothing / 
                                       (sum(self.word_probs[c].values()) * len(self.vocabulary)))
                    
            log_posteriors[c] = log_prob
            
        # Convert to normalized probabilities for interpretation
        max_log = max(log_posteriors.values())
        posteriors = {
            c: np.exp(lp - max_log) 
            for c, lp in log_posteriors.items()
        }
        total = sum(posteriors.values())
        posteriors = {c: p/total for c, p in posteriors.items()}
        
        predicted_class = max(log_posteriors, key=log_posteriors.get)
        return predicted_class, posteriors
 
 
# Demonstration: Training, Generation, and Classification
training_docs = [
    ("great product love it highly recommend", "positive"),
    ("excellent quality best purchase ever", "positive"),
    ("amazing value wonderful experience fantastic", "positive"),
    ("terrible waste of money avoid", "negative"),
    ("horrible quality worst experience ever", "negative"),
    ("disappointing product poor service bad", "negative"),
]
 
documents = [doc for doc, _ in training_docs]
labels = [label for _, label in training_docs]
 
model = GenerativeTextModel(smoothing=1.0)
model.fit(documents, labels)
 
print("Learned Class Priors:")
for c, prior in model.class_priors.items():
    print(f"  P({c}) = {prior:.4f}")
 
print("
Top Words per Class:")
for c in model.classes:
    top_words = sorted(model.word_probs[c].items(), 
                       key=lambda x: x[1], reverse=True)[:5]
    print(f"  {c}: {', '.join(w for w, _ in top_words)}")
 
print("
Generating Sample Documents:")
for c in model.classes:
    synthetic = model.generate_document(c, length=8)
    print(f"  {c}: '{synthetic}'")
 
print("
Classifying New Documents:")
test_docs = [
    "this is an amazing product",
    "terrible quality very disappointed",
    "good value but could be better"
]
for doc in test_docs:
    pred, probs = model.classify(doc)
    print(f"  '{doc[:40]}...'")
    print(f"    -> {pred} (positive: {probs.get('positive', 0):.3f}, "
          f"negative: {probs.get('negative', 0):.3f})")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257

import re
import numpy as np
from collections import Counter, defaultdict
from typing import List, Dict, Tuple, Optional
import string
 
class TextPreprocessor:
    """
    Complete text preprocessing pipeline.
    
    Handles the messy reality of text data: HTML, unicode,
    punctuation, case normalization, and more.
    """
    
    # Common English stopwords
    STOPWORDS = {
        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
        'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare',
        'ought', 'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by',
        'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above',
        'below', 'between', 'under', 'again', 'further', 'then', 'once',
        'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each', 'few',
        'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only',
        'own', 'same', 'so', 'than', 'too', 'very', 'just', 'and', 'but',
        'if', 'or', 'because', 'until', 'while', 'this', 'that', 'these',
        'those', 'am', 'it', 'its', 'itself', 'i', 'me', 'my', 'myself',
        'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself',
        'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself',
        'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who'
    }
    
    def __init__(self, 
                 lowercase: bool = True,
                 remove_stopwords: bool = True,
                 remove_punctuation: bool = True,
                 remove_numbers: bool = False,
                 min_word_length: int = 2):
        self.lowercase = lowercase
        self.remove_stopwords = remove_stopwords
        self.remove_punctuation = remove_punctuation
        self.remove_numbers = remove_numbers
        self.min_word_length = min_word_length
        
    def clean_html(self, text: str) -> str:
        """Remove HTML tags."""
        return re.sub(r'<[^>]+>', ' ', text)
    
    def clean_urls(self, text: str) -> str:
        """Remove URLs."""
        return re.sub(r'http\S+|www\S+', ' ', text)
    
    def normalize_unicode(self, text: str) -> str:
        """Normalize unicode characters."""
        # Replace common unicode characters with ASCII equivalents
        replacements = {
            '"': '"', '"': '"', ''': "'", ''': "'",
            '—': '-', '–': '-', '…': '...'
        }
        for unicode_char, ascii_char in replacements.items():
            text = text.replace(unicode_char, ascii_char)
        return text
    
    def process(self, text: str) -> List[str]:
        """Full preprocessing pipeline."""
        # Clean
        text = self.clean_html(text)
        text = self.clean_urls(text)
        text = self.normalize_unicode(text)
        
        # Lowercase
        if self.lowercase:
            text = text.lower()
            
        # Tokenize (simple whitespace + punctuation splitting)
        if self.remove_punctuation:
            text = text.translate(str.maketrans('', '', string.punctuation))
        tokens = text.split()
        
        # Filter
        if self.remove_numbers:
            tokens = [t for t in tokens if not t.isdigit()]
        if self.remove_stopwords:
            tokens = [t for t in tokens if t not in self.STOPWORDS]
        tokens = [t for t in tokens if len(t) >= self.min_word_length]
        
        return tokens
 
 
class NaiveBayesTextClassifier:
    """
    Complete Naive Bayes text classifier with preprocessing.
    
    Supports both Multinomial and Bernoulli variants.
    """
    
    def __init__(self, 
                 model_type: str = 'multinomial',
                 smoothing: float = 1.0,
                 preprocessor: Optional[TextPreprocessor] = None):
        """
        Args:
            model_type: 'multinomial' or 'bernoulli'
            smoothing: Laplace smoothing parameter
            preprocessor: TextPreprocessor instance (creates default if None)
        """
        self.model_type = model_type
        self.smoothing = smoothing
        self.preprocessor = preprocessor or TextPreprocessor()
        
        self.classes_: List[str] = []
        self.class_log_prior_: Dict[str, float] = {}
        self.feature_log_prob_: Dict[str, Dict[str, float]] = {}
        self.vocabulary_: Dict[str, int] = {}
        
    def fit(self, documents: List[str], labels: List[str]) -> 'NaiveBayesTextClassifier':
        """Fit the model on labeled documents."""
        # Preprocess all documents
        tokenized_docs = [self.preprocessor.process(doc) for doc in documents]
        
        # Build vocabulary
        all_words = set()
        for tokens in tokenized_docs:
            all_words.update(tokens)
        self.vocabulary_ = {word: idx for idx, word in enumerate(sorted(all_words))}
        vocab_size = len(self.vocabulary_)
        
        # Count classes
        self.classes_ = list(set(labels))
        class_counts = Counter(labels)
        n_docs = len(labels)
        
        # Compute log priors
        self.class_log_prior_ = {
            c: np.log(count / n_docs)
            for c, count in class_counts.items()
        }
        
        # Compute feature probabilities per class
        self.feature_log_prob_ = {}
        
        for c in self.classes_:
            # Get documents in this class
            class_docs = [
                tokens for tokens, label in zip(tokenized_docs, labels)
                if label == c
            ]
            
            if self.model_type == 'multinomial':
                # Count total word occurrences
                word_counts = Counter()
                for tokens in class_docs:
                    word_counts.update(tokens)
                total_words = sum(word_counts.values())
                
                # P(w|c) = (count(w,c) + α) / (Σcount + α|V|)
                self.feature_log_prob_[c] = {
                    word: np.log(
                        (word_counts.get(word, 0) + self.smoothing) /
                        (total_words + self.smoothing * vocab_size)
                    )
                    for word in self.vocabulary_
                }
                
            else:  # bernoulli
                # Count documents containing each word
                doc_word_counts = Counter()
                for tokens in class_docs:
                    doc_word_counts.update(set(tokens))  # Unique per doc
                n_class_docs = len(class_docs)
                
                # P(w|c) = (count_docs_with_w + α) / (n_docs_in_class + 2α)
                self.feature_log_prob_[c] = {}
                for word in self.vocabulary_:
                    p_word = (doc_word_counts.get(word, 0) + self.smoothing) /                              (n_class_docs + 2 * self.smoothing)
                    self.feature_log_prob_[c][word] = (
                        np.log(p_word),       # log P(w=1|c)
                        np.log(1 - p_word)    # log P(w=0|c)
                    )
                    
        return self
    
    def predict_log_proba(self, document: str) -> Dict[str, float]:
        """Compute log probabilities for each class."""
        tokens = self.preprocessor.process(document)
        
        log_probs = {}
        for c in self.classes_:
            log_prob = self.class_log_prior_[c]
            
            if self.model_type == 'multinomial':
                # Sum log probabilities of observed words
                for token in tokens:
                    if token in self.feature_log_prob_[c]:
                        log_prob += self.feature_log_prob_[c][token]
            else:  # bernoulli
                # Consider presence AND absence of vocabulary words
                observed_words = set(tokens)
                for word in self.vocabulary_:
                    log_p_present, log_p_absent = self.feature_log_prob_[c][word]
                    if word in observed_words:
                        log_prob += log_p_present
                    else:
                        log_prob += log_p_absent
                        
            log_probs[c] = log_prob
            
        return log_probs
    
    def predict(self, document: str) -> str:
        """Predict the class of a document."""
        log_probs = self.predict_log_proba(document)
        return max(log_probs, key=log_probs.get)
    
    def predict_batch(self, documents: List[str]) -> List[str]:
        """Predict classes for multiple documents."""
        return [self.predict(doc) for doc in documents]
 
 
# Example: Spam Classification
print("=" * 60)
print("SPAM CLASSIFICATION EXAMPLE")
print("=" * 60)
 
training_data = [
    ("Get rich quick! Make money fast with this amazing offer", "spam"),
    ("Congratulations! You've won a free lottery prize click now", "spam"),
    ("LIMITED TIME: Buy now and get 90% discount on watches", "spam"),
    ("Your order has been shipped and will arrive Tuesday", "ham"),
    ("Meeting scheduled for 3pm tomorrow in conference room B", "ham"),
    ("Please review the attached quarterly report before Friday", "ham"),
    ("Free trial! No credit card required claim your prize", "spam"),
    ("Your password has been reset successfully", "ham"),
]
 
train_docs = [doc for doc, _ in training_data]
train_labels = [label for _, label in training_data]
 
# Train classifier
clf = NaiveBayesTextClassifier(model_type='multinomial', smoothing=1.0)
clf.fit(train_docs, train_labels)
 
# Test on new documents
test_docs = [
    "Claim your free prize now! Limited time offer",
    "Please find the meeting notes attached",
    "You've been selected as a lucky winner click here",
    "The project deadline has been extended to next week"
]
 
print("
Predictions:")
for doc in test_docs:
    pred = clf.predict(doc)
    log_probs = clf.predict_log_proba(doc)
    print(f"  '{doc[:50]}...'")
    print(f"    -> {pred.upper()} (log-odds: {log_probs['spam'] - log_probs['ham']:.2f})")