Index: backends/remote/remote-database.cc =================================================================== --- backends/remote/remote-database.cc (revision 15226) +++ backends/remote/remote-database.cc (working copy) @@ -147,10 +147,13 @@ 0)); vector & items = tlist->items; + string term = prefix; char type; while ((type = get_message(message)) == REPLY_METADATAKEYLIST) { NetworkTermListItem item; - item.tname = message; + term.resize(size_t((unsigned char)message[0])); + term.append(message, 1, string::npos); + item.tname = term; items.push_back(item); } if (type != REPLY_DONE) { @@ -217,6 +220,7 @@ 0)); vector & items = tlist->items; + string term = prefix; string message; char type; while ((type = get_message(message)) == REPLY_ALLTERMS) { @@ -224,7 +228,9 @@ const char * p = message.data(); const char * p_end = p + message.size(); item.termfreq = decode_length(&p, p_end, false); - item.tname.assign(p, p_end); + term.resize(size_t((unsigned char)*p++)); + term.append(p, p_end); + item.tname = term; items.push_back(item); } if (type != REPLY_DONE) { Index: common/remoteprotocol.h =================================================================== --- common/remoteprotocol.h (revision 15226) +++ common/remoteprotocol.h (working copy) @@ -44,7 +44,7 @@ // 33: 1.1.3 Support for passing matchspies over the remote connection. // 34: 1.1.4 Support for metadata over with remote databases. // 35: 1.1.5 Support for add_spelling() and remove_spelling(). -// 35.1: 1.2.4 Support for metadata_keys_begin(). +// 35.1: 1.2.4 Support for metadata_keys_begin(); smaller termlist encodings. #define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 35 #define XAPIAN_REMOTE_PROTOCOL_MINOR_VERSION 1 @@ -54,7 +54,7 @@ * pointers in net/remoteserver.cc too. */ enum message_type { - MSG_ALLTERMS, // All Terms + MSG_ALLTERMS_PRE124, // All Terms for Xapian < 1.2.4 MSG_COLLFREQ, // Get Collection Frequency MSG_DOCUMENT, // Get Document MSG_TERMEXISTS, // Term Exists? @@ -83,6 +83,7 @@ MSG_GETMSET, // Get MSet MSG_SHUTDOWN, // Shutdown MSG_METADATAKEYLIST, // Iterator for metadata keys + MSG_ALLTERMS, // All Terms MSG_MAX }; @@ -91,7 +92,7 @@ REPLY_GREETING, // Greeting REPLY_EXCEPTION, // Exception REPLY_DONE, // Done sending list - REPLY_ALLTERMS, // All Terms + REPLY_ALLTERMS_PRE124, // All Terms for Xapian < 1.2.4 REPLY_COLLFREQ, // Get Collection Frequency REPLY_DOCDATA, // Get Document REPLY_TERMDOESNTEXIST, // Term Doesn't Exist @@ -110,6 +111,7 @@ REPLY_RESULTS, // Results (MSet) REPLY_METADATA, // Metadata REPLY_METADATAKEYLIST, // Iterator for metadata keys + REPLY_ALLTERMS, // All Terms REPLY_MAX }; Index: common/remoteserver.h =================================================================== --- common/remoteserver.h (revision 15226) +++ common/remoteserver.h (working copy) @@ -83,6 +83,9 @@ RemoteConnection::send_message(type_as_char, message, end_time); } + // all terms for clients running version < v1.2.4 + void msg_allterms_pre124(const std::string & message); + // all terms void msg_allterms(const std::string & message); Index: net/remoteserver.cc =================================================================== --- net/remoteserver.cc (revision 15227) +++ net/remoteserver.cc (working copy) @@ -39,6 +39,7 @@ #include "serialise.h" #include "serialise-double.h" #include "str.h" +#include "stringutils.h" #include "weightinternal.h" /// Class to throw when we receive the connection closing message. @@ -157,7 +158,7 @@ * don't correspond to dispatch actions. */ static const dispatch_func dispatch[] = { - &RemoteServer::msg_allterms, + &RemoteServer::msg_allterms_pre124, &RemoteServer::msg_collfreq, &RemoteServer::msg_document, &RemoteServer::msg_termexists, @@ -186,6 +187,7 @@ 0, // MSG_GETMSET - used during a conversation. 0, // MSG_SHUTDOWN - handled by get_message(). &RemoteServer::msg_openmetadatakeylist, + &RemoteServer::msg_allterms, }; string message; @@ -231,7 +233,7 @@ } void -RemoteServer::msg_allterms(const string &message) +RemoteServer::msg_allterms_pre124(const string &message) { const string & prefix = message; @@ -239,7 +241,30 @@ for (Xapian::TermIterator t = db->allterms_begin(prefix); t != end; ++t) { string item = encode_length(t.get_termfreq()); item += *t; - send_message(REPLY_ALLTERMS, item); + send_message(REPLY_ALLTERMS_PRE124, item); + } + + send_message(REPLY_DONE, string()); +} + +void +RemoteServer::msg_allterms(const string &message) +{ + string prev = message; + string reply; + + const string & prefix = message; + const Xapian::TermIterator end = db->allterms_end(prefix); + for (Xapian::TermIterator t = db->allterms_begin(prefix); t != end; ++t) { + if (rare(prev.size() > 255)) + prev.resize(255); + const string & v = *t; + size_t reuse = common_prefix_length(prev, v); + reply = encode_length(t.get_termfreq()); + reply.append(1, char(reuse)); + reply.append(v, reuse, string::npos); + send_message(REPLY_ALLTERMS, reply); + prev = v; } send_message(REPLY_DONE, string()); @@ -663,12 +688,22 @@ void RemoteServer::msg_openmetadatakeylist(const string & message) { - const Xapian::TermIterator end = db->metadata_keys_end(message); - Xapian::TermIterator t = db->metadata_keys_begin(message); + string prev = message; + string reply; + + const string & prefix = message; + const Xapian::TermIterator end = db->metadata_keys_end(prefix); + Xapian::TermIterator t = db->metadata_keys_begin(prefix); for (; t != end; ++t) { - send_message(REPLY_METADATAKEYLIST, *t); + if (rare(prev.size() > 255)) + prev.resize(255); + const string & v = *t; + size_t reuse = common_prefix_length(prev, v); + reply.assign(1, char(reuse)); + reply.append(v, reuse, string::npos); + send_message(REPLY_METADATAKEYLIST, reply); + prev = v; } - send_message(REPLY_DONE, string()); }