Branch data Line data Source code
1 : : /* api_valuestats.cc: tests of the value statistics functions.
2 : : *
3 : : * Copyright 2008 Lemur Consulting Ltd
4 : : * Copyright 2008,2009 Olly Betts
5 : : *
6 : : * This program is free software; you can redistribute it and/or
7 : : * modify it under the terms of the GNU General Public License as
8 : : * published by the Free Software Foundation; either version 2 of the
9 : : * License, or (at your option) any later version.
10 : : *
11 : : * This program is distributed in the hope that it will be useful,
12 : : * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : : * GNU General Public License for more details.
15 : : *
16 : : * You should have received a copy of the GNU General Public License
17 : : * along with this program; if not, write to the Free Software
18 : : * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
19 : : * USA
20 : : */
21 : :
22 : : #include <config.h>
23 : :
24 : : #include "api_valuestats.h"
25 : :
26 : : #include <xapian.h>
27 : : #include "testsuite.h"
28 : : #include "testutils.h"
29 : :
30 : : #include "apitest.h"
31 : :
32 : : using namespace std;
33 : :
34 : : // #######################################################################
35 : : // # Tests start here
36 : :
37 : : /// Test of value statistics methods.
38 : 7 : DEFINE_TESTCASE(valuestats1, writable && valuestats) {
39 : 7 : Xapian::WritableDatabase db_w = get_writable_database();
40 : :
41 : : // Check that counts are initially zero.
42 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 0);
43 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
44 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
45 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 0);
46 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
47 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
48 : :
49 : 7 : Xapian::Document doc;
50 : 7 : doc.add_value(0, "hello");
51 : :
52 : : // Check that statistics for the correct value slot increase when document
53 : : // is added. (Check slot 1 first, so that cache invalidation of the last
54 : : // slot read also gets checked.)
55 : 7 : db_w.add_document(doc);
56 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 0);
57 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
58 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
59 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 1);
60 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
61 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
62 : :
63 : : // Check that statistics work correctly when second document is added.
64 : 7 : doc = Xapian::Document();
65 : 7 : doc.add_value(0, "world");
66 : 7 : doc.add_value(1, "cheese");
67 : 7 : db_w.replace_document(2, doc);
68 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 2);
69 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
70 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
71 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 1);
72 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
73 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
74 : :
75 : : // Deleting a document affects the count, but not the bounds.
76 : 7 : db_w.delete_document(1);
77 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 1);
78 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
79 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
80 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 1);
81 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
82 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
83 : :
84 : : // Deleting all the documents returns the bounds to their original value.
85 : 7 : db_w.delete_document(2);
86 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 0);
87 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
88 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
89 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 0);
90 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
91 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
92 : :
93 : : // Adding a document with a value in one of the old slots should still
94 : : // end up with tight bounds on it.
95 : 7 : doc = Xapian::Document();
96 : 7 : doc.add_value(1, "newval");
97 : 7 : db_w.replace_document(2, doc);
98 [ - + # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(1), 1);
99 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
100 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
101 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_freq(0), 0);
102 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
103 [ - + ][ # # ]: 7 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
104 : :
105 : 7 : return true;
106 : : }
107 : :
108 : : /// Test that value statistics stuff obeys transactions.
109 : 6 : DEFINE_TESTCASE(valuestats2, transactions && valuestats) {
110 : 6 : Xapian::WritableDatabase db_w = get_writable_database();
111 : 6 : Xapian::Database db = get_writable_database_as_database();
112 : :
113 : : // Check that counts are initially zero.
114 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 0);
115 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
116 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
117 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 0);
118 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
119 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
120 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(0), 0);
121 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(0), "");
122 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(0), "");
123 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), 0);
124 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "");
125 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "");
126 : :
127 : 6 : Xapian::Document doc;
128 : 6 : doc.add_value(0, "hello");
129 : :
130 : : // Check that statistics for the correct value slot increase when document
131 : : // is added. (Check slot 1 first, so that cache invalidation of the last
132 : : // slot read also gets checked.)
133 : 6 : db_w.add_document(doc);
134 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 0);
135 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
136 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
137 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 1);
138 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
139 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
140 : :
141 : : // The readonly database shouldn't change, though.
142 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), 0);
143 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "");
144 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "");
145 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(0), 0);
146 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(0), "");
147 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(0), "");
148 : :
149 : : // Check that statistics work correctly when second document is added.
150 : 6 : doc = Xapian::Document();
151 : 6 : doc.add_value(0, "world");
152 : 6 : doc.add_value(1, "cheese");
153 : 6 : db_w.replace_document(2, doc);
154 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 2);
155 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
156 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
157 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 1);
158 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
159 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
160 : :
161 : : // The readonly database shouldn't change, though.
162 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(0), 0);
163 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(0), "");
164 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(0), "");
165 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), 0);
166 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "");
167 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "");
168 : :
169 : : // Check that readonly database catches up when a commit is done.
170 : 6 : db_w.commit();
171 : 6 : db.reopen();
172 [ - + # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), 1);
173 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "cheese");
174 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "cheese");
175 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(0), 2);
176 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(0), "hello");
177 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(0), "world");
178 : :
179 : : // Deleting a document affects the count, but not the bounds.
180 : 6 : db_w.delete_document(1);
181 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 1);
182 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
183 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
184 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 1);
185 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
186 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
187 : :
188 : : // Deleting all the documents returns the bounds to their original value.
189 : 6 : db_w.delete_document(2);
190 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 0);
191 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
192 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
193 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 0);
194 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "");
195 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "");
196 : :
197 : : // Adding a document with a value in one of the old slots should still
198 : : // end up with tight bounds on it.
199 : 6 : doc = Xapian::Document();
200 : 6 : doc.add_value(1, "newval");
201 : 6 : db_w.replace_document(2, doc);
202 [ - + # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(1), 1);
203 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
204 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
205 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_freq(0), 0);
206 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_lower_bound(0), "");
207 [ - + ][ # # ]: 6 : TEST_EQUAL(db_w.get_value_upper_bound(0), "");
208 : :
209 : : // Check that a readonly database gets the right statistics, too.
210 : 6 : db_w.commit();
211 : 6 : db.reopen();
212 [ - + # # ]: 6 : TEST_EQUAL(db.get_value_freq(0), 0);
213 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(0), "");
214 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(0), "");
215 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), 1);
216 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "newval");
217 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "newval");
218 : :
219 : 6 : return true;
220 : : }
221 : :
222 : : /// Test reading value statistics from prebuilt databases.
223 : 9 : DEFINE_TESTCASE(valuestats3, valuestats) {
224 : 9 : Xapian::Database db = get_database("apitest_simpledata");
225 : :
226 [ - + # # ]: 9 : TEST_EQUAL(db.get_value_freq(1), 6);
227 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(1), "h");
228 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(1), "n");
229 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(2), 6);
230 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(2), "d");
231 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(2), "i");
232 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(3), 6);
233 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(3), " ");
234 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(3), "s");
235 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(4), 6);
236 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(4), " ");
237 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(4), "y");
238 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(5), 6);
239 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(5), "e");
240 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(5), "p");
241 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(6), 6);
242 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(6), "a");
243 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(6), "t");
244 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(7), 6);
245 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(7), " ");
246 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(7), "r");
247 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(8), 6);
248 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(8), "a");
249 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(8), "t");
250 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(9), 6);
251 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(9), " ");
252 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(9), "n");
253 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(10), 6);
254 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(10), "e");
255 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(10), "w");
256 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_freq(11), 6);
257 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_lower_bound(11), "\xb9P");
258 [ - + ][ # # ]: 9 : TEST_EQUAL(db.get_value_upper_bound(11), "\xc7\x04");
259 : :
260 : 9 : return true;
261 : : }
262 : :
263 : 6 : DEFINE_TESTCASE(valuestats4, transactions && valuestats) {
264 : 6 : const size_t FLUSH_THRESHOLD = 10000;
265 : : {
266 : 6 : Xapian::WritableDatabase db_w = get_writable_database();
267 : 6 : Xapian::Document doc;
268 : 6 : doc.add_value(1, "test");
269 [ + + ]: 60006 : for (size_t i = 0; i < FLUSH_THRESHOLD; ++i) {
270 : 60000 : db_w.add_document(doc);
271 : : }
272 : :
273 : 6 : Xapian::Database db = get_writable_database_as_database();
274 : : // Check that we had an automatic-commit.
275 [ - + # # ]: 6 : TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
276 : : // Check that the value stats are there.
277 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
278 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "test");
279 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "test");
280 : :
281 : 6 : db_w.begin_transaction();
282 : 6 : doc.add_value(1, "umbrella");
283 : 6 : db_w.cancel_transaction();
284 : : }
285 : :
286 : : {
287 : 6 : Xapian::Database db = get_writable_database_as_database();
288 : : // Check that we had an automatic-commit.
289 [ - + # # ]: 6 : TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
290 : : // Check that the value stats are there.
291 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
292 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_lower_bound(1), "test");
293 [ - + ][ # # ]: 6 : TEST_EQUAL(db.get_value_upper_bound(1), "test");
294 : : }
295 : :
296 : 6 : return true;
297 : : }
298 : :
299 : : /// Regression test for bug fixed in 1.1.1 which led to incorrect valuestats.
300 : 1 : DEFINE_TESTCASE(valuestats5, !backend) {
301 : 1 : Xapian::Document doc;
302 : 1 : doc.add_value(0, "zero");
303 : 1 : doc.add_value(1, "one");
304 : 1 : doc.add_value(2, "two");
305 : 1 : doc.add_value(3, "three");
306 : 1 : doc.add_value(4, "");
307 : 1 : doc.add_value(5, "five");
308 : 1 : doc.remove_value(3);
309 : 1 : doc.add_value(1, "");
310 : :
311 : : // Check that we don't have any empty values reported.
312 : 1 : size_t c = 0;
313 : 1 : Xapian::ValueIterator v = doc.values_begin();
314 [ + + ]: 4 : while (v != doc.values_end()) {
315 [ - + ][ # # ]: 3 : TEST(!(*v).empty());
316 : 3 : ++c;
317 : 3 : ++v;
318 : : }
319 [ - + ][ # # ]: 1 : TEST_EQUAL(c, 3); // 0, 2, 5
320 : :
321 : 1 : return true;
322 : : }
|
