git.cworth.org Git - notmuch/blob - lib/parse-sexp.cc

   1 #include "database-private.h"
   2
   3 #if HAVE_SFSEXP
   4 #include "sexp.h"
   5 #include "unicode-util.h"
   6
   7 /* _sexp is used for file scope symbols to avoid clashing with
   8  * definitions from sexp.h */
   9
  10 typedef struct {
  11     const char *name;
  12     const sexp_t *sx;
  13 } _sexp_binding_t;
  14
  15 typedef enum {
  16     SEXP_FLAG_NONE      = 0,
  17     SEXP_FLAG_FIELD     = 1 << 0,
  18     SEXP_FLAG_BOOLEAN   = 1 << 1,
  19     SEXP_FLAG_SINGLE    = 1 << 2,
  20     SEXP_FLAG_WILDCARD  = 1 << 3,
  21     SEXP_FLAG_REGEX     = 1 << 4,
  22     SEXP_FLAG_DO_REGEX  = 1 << 5,
  23     SEXP_FLAG_EXPAND    = 1 << 6,
  24     SEXP_FLAG_DO_EXPAND = 1 << 7,
  25     SEXP_FLAG_ORPHAN    = 1 << 8,
  26 } _sexp_flag_t;
  27
  28 /*
  29  * define bitwise operators to hide casts */
  30
  31 inline _sexp_flag_t
  32 operator| (_sexp_flag_t a, _sexp_flag_t b)
  33 {
  34     return static_cast<_sexp_flag_t>(
  35         static_cast<unsigned>(a) | static_cast<unsigned>(b));
  36 }
  37
  38 inline _sexp_flag_t
  39 operator& (_sexp_flag_t a, _sexp_flag_t b)
  40 {
  41     return static_cast<_sexp_flag_t>(
  42         static_cast<unsigned>(a) & static_cast<unsigned>(b));
  43 }
  44
  45 typedef struct  {
  46     const char *name;
  47     Xapian::Query::op xapian_op;
  48     Xapian::Query initial;
  49     _sexp_flag_t flags;
  50 } _sexp_prefix_t;
  51
  52 static _sexp_prefix_t prefixes[] =
  53 {
  54     { "and",            Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  55       SEXP_FLAG_NONE },
  56     { "attachment",     Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  57       SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
  58     { "body",           Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  59       SEXP_FLAG_FIELD },
  60     { "from",           Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  61       SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  62     { "folder",         Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  63       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  64     { "id",             Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  65       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX },
  66     { "infix",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
  67       SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN },
  68     { "is",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  69       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  70     { "matching",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  71       SEXP_FLAG_DO_EXPAND },
  72     { "mid",            Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  73       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX },
  74     { "mimetype",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  75       SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
  76     { "not",            Xapian::Query::OP_AND_NOT,      Xapian::Query::MatchAll,
  77       SEXP_FLAG_NONE },
  78     { "of",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  79       SEXP_FLAG_DO_EXPAND },
  80     { "or",             Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  81       SEXP_FLAG_NONE },
  82     { "path",           Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  83       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX },
  84     { "property",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  85       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  86     { "query",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchNothing,
  87       SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN },
  88     { "regex",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
  89       SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX },
  90     { "rx",             Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
  91       SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX },
  92     { "starts-with",    Xapian::Query::OP_WILDCARD,     Xapian::Query::MatchAll,
  93       SEXP_FLAG_SINGLE },
  94     { "subject",        Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  95       SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  96     { "tag",            Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
  97       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
  98     { "thread",         Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
  99       SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
 100     { "to",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
 101       SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
 102     { }
 103 };
 104
 105 static notmuch_status_t _sexp_to_xapian_query (notmuch_database_t *notmuch,
 106                                                const _sexp_prefix_t *parent,
 107                                                const _sexp_binding_t *env,
 108                                                const sexp_t *sx,
 109                                                Xapian::Query &output);
 110
 111 static notmuch_status_t
 112 _sexp_combine_query (notmuch_database_t *notmuch,
 113                      const _sexp_prefix_t *parent,
 114                      const _sexp_binding_t *env,
 115                      Xapian::Query::op operation,
 116                      Xapian::Query left,
 117                      const sexp_t *sx,
 118                      Xapian::Query &output)
 119 {
 120     Xapian::Query subquery;
 121
 122     notmuch_status_t status;
 123
 124     /* if we run out elements, return accumulator */
 125
 126     if (! sx) {
 127         output = left;
 128         return NOTMUCH_STATUS_SUCCESS;
 129     }
 130
 131     status = _sexp_to_xapian_query (notmuch, parent, env, sx, subquery);
 132     if (status)
 133         return status;
 134
 135     return _sexp_combine_query (notmuch,
 136                                 parent,
 137                                 env,
 138                                 operation,
 139                                 Xapian::Query (operation, left, subquery),
 140                                 sx->next, output);
 141 }
 142
 143 static notmuch_status_t
 144 _sexp_parse_phrase (std::string term_prefix, const char *phrase, Xapian::Query &output)
 145 {
 146     Xapian::Utf8Iterator p (phrase);
 147     Xapian::Utf8Iterator end;
 148     std::vector<std::string> terms;
 149
 150     while (p != end) {
 151         Xapian::Utf8Iterator start;
 152         while (p != end && ! Xapian::Unicode::is_wordchar (*p))
 153             p++;
 154
 155         if (p == end)
 156             break;
 157
 158         start = p;
 159
 160         while (p != end && Xapian::Unicode::is_wordchar (*p))
 161             p++;
 162
 163         if (p != start) {
 164             std::string word (start, p);
 165             word = Xapian::Unicode::tolower (word);
 166             terms.push_back (term_prefix + word);
 167         }
 168     }
 169     output = Xapian::Query (Xapian::Query::OP_PHRASE, terms.begin (), terms.end ());
 170     return NOTMUCH_STATUS_SUCCESS;
 171 }
 172
 173 static notmuch_status_t
 174 _sexp_parse_wildcard (notmuch_database_t *notmuch,
 175                       const _sexp_prefix_t *parent,
 176                       unused(const _sexp_binding_t *env),
 177                       std::string match,
 178                       Xapian::Query &output)
 179 {
 180
 181     std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
 182
 183     if (parent && ! (parent->flags & SEXP_FLAG_WILDCARD)) {
 184         _notmuch_database_log (notmuch, "'%s' does not support wildcard queries\n", parent->name);
 185         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 186     }
 187
 188     output = Xapian::Query (Xapian::Query::OP_WILDCARD,
 189                             term_prefix + Xapian::Unicode::tolower (match));
 190     return NOTMUCH_STATUS_SUCCESS;
 191 }
 192
 193 static notmuch_status_t
 194 _sexp_parse_one_term (notmuch_database_t *notmuch, std::string term_prefix, const sexp_t *sx,
 195                       Xapian::Query &output)
 196 {
 197     Xapian::Stem stem = *(notmuch->stemmer);
 198
 199     if (sx->aty == SEXP_BASIC && unicode_word_utf8 (sx->val)) {
 200         std::string term = Xapian::Unicode::tolower (sx->val);
 201
 202         output = Xapian::Query ("Z" + term_prefix + stem (term));
 203         return NOTMUCH_STATUS_SUCCESS;
 204     } else {
 205         return _sexp_parse_phrase (term_prefix, sx->val, output);
 206     }
 207
 208 }
 209
 210 notmuch_status_t
 211 _sexp_parse_regex (notmuch_database_t *notmuch,
 212                    const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
 213                    unused(const _sexp_binding_t *env),
 214                    std::string val, Xapian::Query &output)
 215 {
 216     if (! parent) {
 217         _notmuch_database_log (notmuch, "illegal '%s' outside field\n",
 218                                prefix->name);
 219         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 220     }
 221
 222     if (! (parent->flags & SEXP_FLAG_REGEX)) {
 223         _notmuch_database_log (notmuch, "'%s' not supported in field '%s'\n",
 224                                prefix->name, parent->name);
 225         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 226     }
 227
 228     std::string msg; /* ignored */
 229
 230     return _notmuch_regexp_to_query (notmuch, Xapian::BAD_VALUENO, parent->name,
 231                                      val, output, msg);
 232 }
 233
 234
 235 static notmuch_status_t
 236 _sexp_expand_query (notmuch_database_t *notmuch,
 237                     const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
 238                     unused(const _sexp_binding_t *env), const sexp_t *sx, Xapian::Query &output)
 239 {
 240     Xapian::Query subquery;
 241     notmuch_status_t status;
 242     std::string msg;
 243
 244     if (! (parent->flags & SEXP_FLAG_EXPAND)) {
 245         _notmuch_database_log (notmuch, "'%s' unsupported inside '%s'\n", prefix->name, parent->name);
 246         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 247     }
 248
 249     status = _sexp_combine_query (notmuch, NULL, NULL, prefix->xapian_op, prefix->initial, sx,
 250                                   subquery);
 251     if (status)
 252         return status;
 253
 254     status = _notmuch_query_expand (notmuch, parent->name, subquery, output, msg);
 255     if (status) {
 256         _notmuch_database_log (notmuch, "error expanding query %s\n", msg.c_str ());
 257     }
 258     return status;
 259 }
 260
 261 static notmuch_status_t
 262 _sexp_parse_infix (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output)
 263 {
 264     try {
 265         output = notmuch->query_parser->parse_query (sx->val, NOTMUCH_QUERY_PARSER_FLAGS);
 266     } catch (const Xapian::QueryParserError &error) {
 267         _notmuch_database_log (notmuch, "Syntax error in infix query: %s\n", sx->val);
 268         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 269     } catch (const Xapian::Error &error) {
 270         if (! notmuch->exception_reported) {
 271             _notmuch_database_log (notmuch,
 272                                    "A Xapian exception occurred parsing query: %s\n",
 273                                    error.get_msg ().c_str ());
 274             _notmuch_database_log_append (notmuch,
 275                                           "Query string was: %s\n",
 276                                           sx->val);
 277             notmuch->exception_reported = true;
 278             return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 279         }
 280     }
 281     return NOTMUCH_STATUS_SUCCESS;
 282 }
 283
 284 static notmuch_status_t
 285 _sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 286                     const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
 287 {
 288     _sexp_prefix_t user_prefix;
 289
 290     user_prefix.name = sx->list->val;
 291     user_prefix.flags = SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD;
 292
 293     if (parent) {
 294         _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n",
 295                                sx->list->val, parent->name);
 296         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 297     }
 298
 299     parent = &user_prefix;
 300
 301     return _sexp_combine_query (notmuch, parent, env, Xapian::Query::OP_AND, Xapian::Query::MatchAll,
 302                                 sx->list->next, output);
 303 }
 304
 305 static notmuch_status_t
 306 maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 307                     const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
 308 {
 309     char *key;
 310     char *expansion = NULL;
 311     notmuch_status_t status;
 312     sexp_t *saved_sexp;
 313     void *local = talloc_new (notmuch);
 314     char *buf;
 315
 316     key = talloc_asprintf (local, "squery.%s", sx->list->val);
 317     if (! key) {
 318         status = NOTMUCH_STATUS_OUT_OF_MEMORY;
 319         goto DONE;
 320     }
 321
 322     status = notmuch_database_get_config (notmuch, key, &expansion);
 323     if (status)
 324         goto DONE;
 325     if (EMPTY_STRING (expansion)) {
 326         status = NOTMUCH_STATUS_IGNORED;
 327         goto DONE;
 328     }
 329
 330     buf = talloc_strdup (local, expansion);
 331     /* XXX TODO: free this memory */
 332     saved_sexp = parse_sexp (buf, strlen (expansion));
 333     if (! saved_sexp) {
 334         _notmuch_database_log (notmuch, "invalid saved s-expression query: '%s'\n", expansion);
 335         status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 336         goto DONE;
 337     }
 338
 339     status =  _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output);
 340
 341   DONE:
 342     if (local)
 343         talloc_free (local);
 344
 345     return status;
 346 }
 347
 348 /* Here we expect the s-expression to be a proper list, with first
 349  * element defining and operation, or as a special case the empty
 350  * list */
 351
 352 static notmuch_status_t
 353 _sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
 354                        const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
 355 {
 356     notmuch_status_t status;
 357
 358     if (sx->ty == SEXP_VALUE) {
 359         std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
 360
 361         if (sx->aty == SEXP_BASIC && strcmp (sx->val, "*") == 0) {
 362             return _sexp_parse_wildcard (notmuch, parent, env, "", output);
 363         }
 364
 365         if (parent && (parent->flags & SEXP_FLAG_BOOLEAN)) {
 366             output = Xapian::Query (term_prefix + sx->val);
 367             return NOTMUCH_STATUS_SUCCESS;
 368         }
 369
 370         if (parent) {
 371             return _sexp_parse_one_term (notmuch, term_prefix, sx, output);
 372         } else {
 373             Xapian::Query accumulator;
 374             for (_sexp_prefix_t *prefix = prefixes; prefix->name; prefix++) {
 375                 if (prefix->flags & SEXP_FLAG_FIELD) {
 376                     Xapian::Query subquery;
 377                     term_prefix = _notmuch_database_prefix (notmuch, prefix->name);
 378                     status = _sexp_parse_one_term (notmuch, term_prefix, sx, subquery);
 379                     if (status)
 380                         return status;
 381                     accumulator = Xapian::Query (Xapian::Query::OP_OR, accumulator, subquery);
 382                 }
 383             }
 384             output = accumulator;
 385             return NOTMUCH_STATUS_SUCCESS;
 386         }
 387     }
 388
 389     /* Empty list */
 390     if (! sx->list) {
 391         output = Xapian::Query::MatchAll;
 392         return NOTMUCH_STATUS_SUCCESS;
 393     }
 394
 395     if (sx->list->ty == SEXP_LIST) {
 396         _notmuch_database_log (notmuch, "unexpected list in field/operation position\n",
 397                                sx->list->val);
 398         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 399     }
 400
 401     status = maybe_saved_squery (notmuch, parent, env, sx, output);
 402     if (status != NOTMUCH_STATUS_IGNORED)
 403         return status;
 404
 405     /* Check for user defined field */
 406     if (_notmuch_string_map_get (notmuch->user_prefix, sx->list->val)) {
 407         return _sexp_parse_header (notmuch, parent, env, sx, output);
 408     }
 409
 410     for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) {
 411         if (strcmp (prefix->name, sx->list->val) == 0) {
 412             if (prefix->flags & SEXP_FLAG_FIELD) {
 413                 if (parent) {
 414                     _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n",
 415                                            prefix->name, parent->name);
 416                     return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 417                 }
 418                 parent = prefix;
 419             }
 420
 421             if (parent && (prefix->flags & SEXP_FLAG_ORPHAN)) {
 422                 _notmuch_database_log (notmuch, "'%s' not supported inside '%s'\n",
 423                                        prefix->name, parent->name);
 424                 return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 425             }
 426
 427             if ((prefix->flags & SEXP_FLAG_SINGLE) &&
 428                 (! sx->list->next || sx->list->next->next || sx->list->next->ty != SEXP_VALUE)) {
 429                 _notmuch_database_log (notmuch, "'%s' expects single atom as argument\n",
 430                                        prefix->name);
 431                 return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 432             }
 433
 434             if (strcmp (prefix->name, "infix") == 0) {
 435                 return _sexp_parse_infix (notmuch, sx->list->next, output);
 436             }
 437
 438             if (strcmp (prefix->name, "query") == 0) {
 439                 return _notmuch_query_name_to_query (notmuch, sx->list->next->val, output);
 440             }
 441
 442             if (prefix->xapian_op == Xapian::Query::OP_WILDCARD)
 443                 return _sexp_parse_wildcard (notmuch, parent, env, sx->list->next->val, output);
 444
 445             if (prefix->flags & SEXP_FLAG_DO_REGEX) {
 446                 return _sexp_parse_regex (notmuch, prefix, parent, env, sx->list->next->val, output);
 447             }
 448
 449             if (prefix->flags & SEXP_FLAG_DO_EXPAND) {
 450                 return _sexp_expand_query (notmuch, prefix, parent, env, sx->list->next, output);
 451             }
 452
 453             return _sexp_combine_query (notmuch, parent, env, prefix->xapian_op, prefix->initial,
 454                                         sx->list->next, output);
 455         }
 456     }
 457
 458     _notmuch_database_log (notmuch, "unknown prefix '%s'\n", sx->list->val);
 459     return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 460 }
 461
 462 notmuch_status_t
 463 _notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr,
 464                                       Xapian::Query &output)
 465 {
 466     const sexp_t *sx = NULL;
 467     char *buf = talloc_strdup (notmuch, querystr);
 468
 469     sx = parse_sexp (buf, strlen (querystr));
 470     if (! sx) {
 471         _notmuch_database_log (notmuch, "invalid s-expression: '%s'\n", querystr);
 472         return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
 473     }
 474
 475     return _sexp_to_xapian_query (notmuch, NULL, NULL, sx, output);
 476 }
 477 #endif