diff --git a/xapian-applications/omega/Makefile.am b/xapian-applications/omega/Makefile.am index 59f44a2..1da1398 100644 --- a/xapian-applications/omega/Makefile.am +++ b/xapian-applications/omega/Makefile.am @@ -70,12 +70,13 @@ EXTRA_DIST =\ xapian-omega.spec AM_CPPFLAGS = \ + -I/usr/include/wv2 \ -DCONFIGFILE_SYSTEM=\"$(sysconfdir)/omega.conf\" \ -DPKGLIBBINDIR=\"$(pkglibbindir)\" AM_CXXFLAGS += $(XAPIAN_CXXFLAGS) pkglibbindir = $(pkglibdir)/bin -pkglibbin_PROGRAMS = omega +pkglibbin_PROGRAMS = omega omindex_wv dist_pkglibbin_SCRIPTS = outlookmsg2html bin_PROGRAMS = omindex scriptindex dist_bin_SCRIPTS = dbi2omega htdig2omega mbox2omega @@ -94,10 +95,11 @@ noinst_HEADERS = omega.h query.h cgiparam.h\ md5.h md5wrap.h xmlparse.h metaxmlparse.h values.h utf8convert.h\ namedentities.h pkglibbindir.h datematchdecider.h sample.h strcasecmp.h\ utf8truncate.h diritor.h runfilter.h freemem.h xpsxmlparse.h transform.h\ - weight.h svgparse.h urlencode.h unixperm.h + weight.h svgparse.h urlencode.h unixperm.h worker.h worker_comms.h wv.h # headers maintained in xapian-core noinst_HEADERS +=\ + common/closefrom.h\ common/gnu_getopt.h\ common/msvc_dirent.h\ common/noreturn.h\ @@ -135,12 +137,16 @@ omindex_SOURCES = omindex.cc myhtmlparse.cc htmlparse.cc\ common/getopt.cc commonhelp.cc utils.cc hashterm.cc loadfile.cc md5.cc\ md5wrap.cc xmlparse.cc metaxmlparse.cc utf8convert.cc sample.cc diritor.cc\ runfilter.cc freemem.cc common/msvc_dirent.cc xpsxmlparse.cc common/str.cc\ - pkglibbindir.cc svgparse.cc urlencode.cc + pkglibbindir.cc svgparse.cc common/closefrom.cc worker.cc worker_comms.cc\ + urlencode.cc if NEED_MKDTEMP omindex_SOURCES += portability/mkdtemp.cc endif omindex_LDADD = $(MAGIC_LIBS) $(XAPIAN_LIBS) +omindex_wv_SOURCES = module.cc worker_comms.cc wv.cc +omindex_wv_LDADD = -lwv2 $(XAPIAN_LIBS) + scriptindex_SOURCES = scriptindex.cc myhtmlparse.cc htmlparse.cc\ common/getopt.cc commonhelp.cc utils.cc hashterm.cc loadfile.cc\ common/safe.cc common/stringutils.cc utf8convert.cc utf8truncate.cc diff --git a/xapian-applications/omega/module.cc b/xapian-applications/omega/module.cc new file mode 100644 index 0000000..ec740c4 --- /dev/null +++ b/xapian-applications/omega/module.cc @@ -0,0 +1,54 @@ +/** @file module.cc + * @brief Worker module for putting text extraction into a separate process. + */ +/* Copyright (C) 2011 Olly Betts + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +#include + +#include "worker_comms.h" +#include "wv.h" + +#include +#include + +using namespace std; + +const int FD = 3; + +// FIXME: Restart filter every N files processed? + +int main() { + string filename; + FILE * f = fdopen(FD, "r+"); + while (true) { + // Read filename. + if (!read_string(f, filename)) break; + + string dump, title, keywords, author; + if (!wv_extract(filename, dump, title, keywords, author)) { + // FIXME: we could persist even if extraction fails... + exit(1); + } + + if (!write_string(f, dump) || + !write_string(f, title) || + !write_string(f, keywords) || + !write_string(f, author)) break; + } +} diff --git a/xapian-applications/omega/omindex.cc b/xapian-applications/omega/omindex.cc index c506fea..4ed64c5 100644 --- a/xapian-applications/omega/omindex.cc +++ b/xapian-applications/omega/omindex.cc @@ -59,6 +59,7 @@ #include "utf8convert.h" #include "utils.h" #include "values.h" +#include "worker.h" #include "xmlparse.h" #include "xpsxmlparse.h" @@ -76,6 +77,8 @@ using namespace std; #define PROG_NAME "omindex" #define PROG_DESC "Index static website data via the filesystem" +Worker wv_worker("omindex_wv"); + static bool skip_duplicates = false; static bool follow_symlinks = false; static bool spelling = false; @@ -626,6 +629,11 @@ index_file(const string &file, const string &url, DirectoryIterator & d, } catch (ReadError) { // It's probably best to index the document even if this fails. } + } else if (mimetype == "application/msword") { + if (!wv_worker.extract(file, dump, title, keywords, author)) { + skip(file, "libwv2 failed"); + return; + } } else if (mimetype == "application/vnd.ms-excel") { string cmd = "xls2csv -c' ' -q0 -dutf-8 " + shell_protect(file); try { @@ -1372,7 +1380,7 @@ main(int argc, char **argv) mime_map["obj"] = "ignore"; mime_map["so"] = "ignore"; - commands["application/msword"] = "antiword -mUTF-8.txt "; + //commands["application/msword"] = "antiword -mUTF-8.txt "; commands["application/vnd.ms-powerpoint"] = "catppt -dutf-8 "; // Looking at the source of wpd2html and wpd2text I think both output // UTF-8, but it's hard to be sure without sample Unicode .wpd files diff --git a/xapian-applications/omega/worker.cc b/xapian-applications/omega/worker.cc new file mode 100644 index 0000000..1632c3d --- /dev/null +++ b/xapian-applications/omega/worker.cc @@ -0,0 +1,142 @@ +/** @file worker.cc + * @brief Class representing worker process. + */ +/* Copyright (C) 2005,2006,2007,2008,2009,2010,2011 Olly Betts + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +#include + +#include "worker.h" + +#include "pkglibbindir.h" +#include "worker_comms.h" + +#include +#include +#include "safeerrno.h" +#include "safefcntl.h" +#include +#include +#include +#include "safesysstat.h" +#include "safeunistd.h" + +#include "closefrom.h" + +using namespace std; + +bool Worker::ignoring_sigpipe = false; + +void +Worker::start_worker_process() +{ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, fds) < 0) { + throw string("socketpair failed: ") + strerror(errno); + } + + child = fork(); + if (child == 0) { + // Child process. + close(fds[0]); + + // Connect pipe to fd 3. Don't use stdin and stdout in case the + // filter library tries to read from stdin or write debug or progress + // info to stdout. + dup2(fds[1], 3); + + // Make sure we don't hang on to open files which may get deleted but + // not have their disk space released until we exit. + closefrom(4); + + // Connect stdin, stdout, stderr to /dev/null. + int devnull = open("/dev/null", O_RDWR); + dup2(devnull, 0); + dup2(devnull, 1); + dup2(devnull, 2); + if (devnull > 3) close(devnull); + + // FIXME: For filters which support a file descriptor as input, we + // could open the file here, and pass the file descriptor across the + // socket to the worker process using sendmsg(). Then the worker + // process could be chroot()-ed to a sandbox directory, which means + // we're reasonably protected from security bugs in the filter. + + const char * mod; + if (filter_module.find('/') == string::npos) { + // Look for unqualified filters in pkglibbindir. + string full_path = get_pkglibbindir(); + full_path += '/'; + full_path += filter_module; + mod = full_path.c_str(); + } else { + mod = filter_module.c_str(); + } + + execl(mod, mod, static_cast(NULL)); + _exit(1); + } + + int fork_errno = errno; + close(fds[1]); + if (child == -1) { + close(fds[0]); + throw string("fork failed: ") + strerror(fork_errno); + } + + f = fdopen(fds[0], "r+"); + + if (!ignoring_sigpipe) { + ignoring_sigpipe = true; + signal(SIGPIPE, SIG_IGN); + } +} + +bool +Worker::extract(const std::string & filename, + std::string & dump, + std::string & title, + std::string & keywords, + std::string & author) +{ + if (f) { + // Check if the worker process is still alive - if it is, waitpid() // + // with WNOHANG returns 0. + int status; + if (waitpid(child, &status, WNOHANG) != 0) { + fclose(f); + f = NULL; + } + } + if (!f) { + start_worker_process(); + } + + if (write_string(f, filename) && + read_string(f, dump) && + read_string(f, title) && + read_string(f, keywords) && + read_string(f, author)) return true; + + fclose(f); + f = NULL; + + int status; + waitpid(child, &status, 0); + return false; +} diff --git a/xapian-applications/omega/worker.h b/xapian-applications/omega/worker.h new file mode 100644 index 0000000..d557ecb --- /dev/null +++ b/xapian-applications/omega/worker.h @@ -0,0 +1,46 @@ +/** @file worker.h + * @brief Class representing worker process. + */ +/* Copyright (C) 2011 Olly Betts + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +#include +#include +#include + +class Worker { + static bool ignoring_sigpipe; + + pid_t child; + + std::FILE * f; + + std::string filter_module; + + void start_worker_process(); + + public: + Worker(const std::string & filter_module_) + : f(NULL), filter_module(filter_module_) { } + + bool extract(const std::string & filename, + std::string & dump, + std::string & title, + std::string & keywords, + std::string & author); +}; diff --git a/xapian-applications/omega/worker_comms.cc b/xapian-applications/omega/worker_comms.cc new file mode 100644 index 0000000..48a0f0e --- /dev/null +++ b/xapian-applications/omega/worker_comms.cc @@ -0,0 +1,103 @@ +/** @file worker_comms.cc + * @brief Communication with worker processes + */ +/* Copyright (C) 2011 Olly Betts + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +#include + +#include "worker_comms.h" + +#include "safeerrno.h" +#include "safeunistd.h" +#include +#include + +using namespace std; + +bool +read_string(FILE * f, string & s) +{ + int ch = getc(f); + if (ch < 0) return false; + size_t len = ch; + if (len >= 253) { + unsigned i = len - 251; + len = 0; + while (i-- > 0) { + ch = getc(f); + if (ch < 0) return false; + len = (len << 8) | ch; + } + } + + s.resize(0); + s.reserve(len); + + char buf[4096]; + + while (len) { + size_t n = fread(buf, 1, min(sizeof(buf), len), f); + if (n == 0) { + // Error or EOF! + return false; + } + s.append(buf, size_t(n)); + len -= n; + } + + return true; +} + +bool +write_string(FILE * f, const string & s) +{ + size_t len = s.size(); + if (len < 253) { + putc(static_cast(len), f); + } else if (len < 0x10000) { + putc(253, f); + putc(static_cast(len >> 8), f); + putc(static_cast(len), f); + } else if (len < 0x1000000) { + putc(254, f); + putc(static_cast(len >> 16), f); + putc(static_cast(len >> 8), f); + putc(static_cast(len), f); + } else { + putc(255, f); + putc(static_cast(len >> 24), f); + putc(static_cast(len >> 16), f); + putc(static_cast(len >> 8), f); + putc(static_cast(len), f); + } + + const char * p = s.data(); + + while (len) { + size_t n = fwrite(p, 1, len, f); + if (n == 0) { + // EOF. + return false; + } + p += n; + len -= n; + } + + return true; +} diff --git a/xapian-applications/omega/worker_comms.h b/xapian-applications/omega/worker_comms.h new file mode 100644 index 0000000..2f19195 --- /dev/null +++ b/xapian-applications/omega/worker_comms.h @@ -0,0 +1,27 @@ +/** @file worker_comms.h + * @brief Communication with worker processes + */ +/* Copyright (C) 2011 Olly Betts + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 + * USA + */ + +#include +#include + +bool read_string(std::FILE * f, std::string & s); + +bool write_string(std::FILE * f, const std::string & s); diff --git a/xapian-applications/omega/wv.cc b/xapian-applications/omega/wv.cc new file mode 100644 index 0000000..084ee60 --- /dev/null +++ b/xapian-applications/omega/wv.cc @@ -0,0 +1,182 @@ +/** @file wv.cc + * @brief Extract text and metadata using libwv2. + * + * Copyright (C) 2010,2011 Olly Betts + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "wv.h" + +#include +#include +#include +#include +#include + +#include + +#include "safefcntl.h" +#include +#include + +using namespace std; +using namespace wvWare; + +class StderrSink { + static int devnull; + + int real_stderr; + + public: + StderrSink() { + if (devnull < 0) { + devnull = open("/dev/null", O_WRONLY); + if (devnull < 0) { + real_stderr = -1; + return; + } + } + cerr << flush; + real_stderr = dup(2); + if (real_stderr >= 0) + dup2(devnull, 2); + } + + ~StderrSink() { + // Restore stderr. + if (real_stderr >= 0) { + cerr << flush; + dup2(real_stderr, 2); + close(real_stderr); + } + } +}; + +int StderrSink::devnull = -1; + +static void +append_utf8(string &s, const UString & us) +{ + for (int i = 0; i < us.length(); ++i) + Xapian::Unicode::append_utf8(s, us[i].unicode()); +} + +class MyTextHandler : public TextHandler { + bool ignore; + + public: + string dump; + string title; + string author; + string keywords; + + MyTextHandler() : ignore(false) { } + + void paragraphEnd(); + + void pictureFound(const PictureFunctor & picture, SharedPtr picf, SharedPtr chp); + + void runOfText(const UString& text, SharedPtr chp); + + void fieldStart(const wvWare::FLD* fld, wvWare::SharedPtr chp); + void fieldEnd(const wvWare::FLD* fld, wvWare::SharedPtr chp); +}; + +void +MyTextHandler::paragraphEnd() +{ + dump += '\n'; + TextHandler::paragraphEnd(); +} + +void +MyTextHandler::pictureFound(const PictureFunctor &, SharedPtr, SharedPtr) +{ + // Ignore pictures. +} + +void +MyTextHandler::runOfText(const UString& text, SharedPtr chp) +{ + if (!ignore) + append_utf8(dump, text); + TextHandler::runOfText(text, chp); +} + +void +MyTextHandler::fieldStart(const FLD* fld, SharedPtr chp) +{ + ignore = true; + TextHandler::fieldStart(fld, chp); +} + +void +MyTextHandler::fieldEnd( const FLD* fld, SharedPtr chp ) +{ + TextHandler::fieldEnd(fld, chp); + ignore = false; +} + +bool +wv_extract(const string & filename, + string & dump, + string & title, + string & keywords, + string & author) +{ + const char * error = NULL; + + try { + // libwv2-4 0.4.2.dfsg.1-1 in Debian and Ubuntu isn't built with NDEBUG + // and so spews huge amounts of debug log to stderr. I've filed a bug + // with a patch to the Debian BTS, and put the patch on Launchpad too, + // but at least until this is applied and the affected versions have + // become obsolete, we want to discard stderr output while libwv2 is + // working. + StderrSink eat_stderr; + + SharedPtr parser(ParserFactory::createParser(filename)); + if (parser && parser->isOk()) { + MyTextHandler text_handler; + parser->setTextHandler(&text_handler); + + if (parser->parse()) { + AssociatedStrings strings(parser->associatedStrings()); + append_utf8(title, strings.title()); + append_utf8(keywords, strings.keywords()); + if (!keywords.empty()) keywords += '\n'; + append_utf8(keywords, strings.subject()); + append_utf8(author, strings.author()); + + swap(dump, text_handler.dump); + } else { + error = "wv2 failed to parse file"; + } + } else { + error = "Failed to initialise wv2 parser"; + } + } catch (...) { + error = "wv2 parser threw an exception"; + } + + if (!error) + return true; + + cerr << error << endl; + return false; +} diff --git a/xapian-applications/omega/wv.h b/xapian-applications/omega/wv.h new file mode 100644 index 0000000..e3d1c40 --- /dev/null +++ b/xapian-applications/omega/wv.h @@ -0,0 +1,33 @@ +/** @file wv.h + * @brief Extract text and metadata using libwv2. + * + * Copyright (C) 2010 Olly Betts + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef OMEGA_INCLUDED_WV_H +#define OMEGA_INCLUDED_WV_H + +#include + +bool +wv_extract(const std::string & filename, + std::string & dump, + std::string & title, + std::string & keywords, + std::string & author); + +#endif // OMEGA_INCLUDED_WV_H