9ed3771f by Adam Heath

Generic solr search model class.

1 parent a96a4437
1 define(function(require) { 1 define(function(require) {
2 'use strict'; 2 'use strict';
3 var _ = require('underscore');
4 var Backbone = require('backbone');
5 var NestedModels = require('backbone-nested-models');
6 //var module = require('module');
7
8 function getField(obj, key) {
9 return obj[key];
10 }
11
12 function mergeStaticProps(startPtr, endPtr, obj, fieldName) {
13 var result = obj;
14 var ptr = startPtr;
15 while (true) {
16 result = _.extend(result, _.result(ptr, fieldName));
17 if (ptr === endPtr) {
18 break;
19 }
20 ptr = ptr.__super__.constructor;
21 }
22 return result;
23 }
24
25 function getItemKeyAccessor(item) {
26 return item.get('key');
27 }
28 var SolrFacets = (function() {
29 var Facets = NestedModels.mixin(Backbone.Model.extend({
30 initialize: function(data, options) {
31 _.each(this.keys(), _.bind(function(facetName) {
32 var facet = this.get(facetName);
33 facet.on('item-change', function() {
34 this.trigger('item-change');
35 }, this);
36 }, this));
37 return Facets.__super__.initialize.apply(this, arguments);
38 },
39 resetSearch: function() {
40 _.each(this.values(), function(facet) {
41 facet.get('items').reset();
42 });
43 },
44 applyFacetResults: function(data) {
45 var facetCounts = getField(data, 'facet_counts');
46 var facetRanges = getField(facetCounts, 'facet_ranges');
47 var facetFields = getField(facetCounts, 'facet_fields');
48 var facetQueries = getField(facetCounts, 'facet_queries');
49 _.each(this.keys(), _.bind(function(facetName) {
50 var facet = this.get(facetName);
51 var type = facet.facetType;
52 var list;
53 var key, newItems = [];
54 var items = facet.get('items');
55 items.invoke('set', 'hidden', true);
56 var valueOverrides = {};
57 var excludeValues = {};
58 function recordIncludeValueIntoExclude(includeValue, key) {
59 excludeValues[includeValue] = true;
60 }
61 switch (type) {
62 case 'year':
63 case 'range':
64 list = facetRanges[facetName].counts;
65 break;
66 case 'field':
67 list = facetFields[facetName];
68 _.each(facet.bins, function(includeValues, key) {
69 var tag = facetName + ':' + key;
70 valueOverrides[key] = facetQueries[tag];
71 _.each(includeValues, recordIncludeValueIntoExclude);
72 });
73 break;
74 }
75 var addNewItem = _.bind(function addNewItem(key, value) {
76 if (valueOverrides[key] !== undefined) {
77 value = valueOverrides[key];
78 if (!value) {
79 return;
80 }
81 } else if (excludeValues[key]) {
82 return;
83 }
84 var item = items.get({id: key});
85 if (item) {
86 item.set({hidden: value === 0, value: value});
87 } else {
88 item = new Item({key: key, value: value});
89 item.on('change:checked', function() {
90 this.trigger('item-change');
91 facet.trigger('item-change');
92 }, this);
93 }
94 newItems.push(item);
95 }, this);
96
97 function checkOther(set, key) {
98 if (set.indexOf(facet.other) !== -1) {
99 addNewItem(key, facetRanges[facetName][key]);
100 }
101 }
102 checkOther(['all', 'before'], 'before');
103
104 _.each(list, function(value, index) {
105 if (index % 2 === 0) {
106 key = value;
107 } else if (value > 0) {
108 addNewItem(key, value);
109 }
110 });
111 switch (type) {
112 case 'field':
113 _.each(facet.bins, function(includeValues, key) {
114 var tag = facetName + ':' + key;
115 addNewItem(key, facetQueries[tag]);
116 });
117 break;
118 }
119 items.set(newItems, {remove: false});
120 }, this));
121 },
122 getFacetFormData: function() {
123 var result = {
124 facet: true,
125 'facet.missing': true,
126 'facet.field': [],
127 'facet.range': [],
128 'facet.query': [],
129 fq: [],
130 };
131 _.each(this.keys(), _.bind(function(facetName) {
132 var facet = this.get(facetName);
133 var queryField = facet.queryField ? facet.queryField : facetName;
134 var type = facet.facetType;
135 var valueFormatter;
136 var facetOptions = {};
137 switch (type) {
138 case 'year':
139 type = 'range';
140 valueFormatter = function(item) {
141 var key = item.get('key');
142 var fromDate, toDate;
143 if (key === 'before') {
144 return '[* TO ' + facet.rangeStart + '-1SECOND]';
145 }
146 fromDate = new Date(key);
147 toDate = new Date(key);
148 toDate.setUTCFullYear(toDate.getUTCFullYear() + 1);
149 return '[' + fromDate.toISOString() + ' TO ' + toDate.toISOString() + ']';
150 };
151 if (facet.other) {
152 facetOptions['facet.range.other'] = facet.other;
153 }
154 break;
155 case 'range':
156 valueFormatter = function(item) {
157 var key = parseInt(item.get('key'));
158 return '[' + key + ' TO ' + (key + facet.rangeGap) + ']';
159 };
160 break;
161 case 'field':
162 valueFormatter = facet.queryValue ? facet.queryValue : getItemKeyAccessor;
163 _.each(facet.bins, function(includeValues, key) {
164 var query = _.map(includeValues, function(includeValue, index) {
165 return queryField + ':' + includeValue;
166 }).join(' OR ');
167 result['facet.query'].push('{!key=' + facetName + ':' + key + ' tag=' + facetName + ':' + key + '}(' + query + ')');
168 });
169 break;
170 }
171 switch (type) {
172 case 'range':
173 facetOptions['facet.range.start'] = facet.rangeStart;
174 facetOptions['facet.range.end'] = facet.rangeEnd;
175 facetOptions['facet.range.gap'] = facet.rangeGap;
176 break;
177 }
178 var facetValues = [];
179 facet.get('items').each(function(item, index) {
180 if (!item.get('hidden') && item.get('checked')) {
181 facetValues.push(valueFormatter(item));
182 }
183 });
184 facetOptions.key = facetName;
185 if (facetValues.length) {
186 facetOptions.ex = facetName;
187 result.fq.push('{!tag=' + facetName + '}' + queryField + ':(' + facetValues.join(' OR ') + ')');
188 } else {
189 facetOptions.tag = facetName;
190 }
191 var facetOptionList = [];
192 _.each(facetOptions, function(value, key) {
193 if (false && key.indexOf('facet.') === 0) {
194 result['f.' + queryField + '.' + key] = value;
195 } else {
196 facetOptionList.push(key + '=' + value);
197 }
198 });
199 result['facet.' + type].push('{!' + facetOptionList.join(' ') + '}' + queryField);
200 }, this));
201 return result;
202 }
203 }));
204 var Sort = Facets.Sort = {
205 standard: function standardSort(a, b) {
206 if (a < b) {
207 return -1;
208 } else if (a > b) {
209 return 1;
210 } else {
211 return 0;
212 }
213 },
214 date: function dateSort(a, b) {
215 return Sort.standard.call(this, a, b);
216 },
217 number: function numberSort(a, b) {
218 return Sort.standard.call(this, parseInt(a), parseInt(b));
219 },
220 };
221 function wrapComparatorForAllBeforeAfter(comparator, doBefore, doAfter) {
222 if (!doBefore && !doAfter) {
223 return comparator;
224 }
225 return function beforeAfterComparator(a, b) {
226 if (doBefore) {
227 if (a === 'before') {
228 return -1;
229 } else if (b === 'before') {
230 return 1;
231 }
232 }
233 if (doAfter) {
234 if (a === 'after') {
235 return 1;
236 } else if (b === 'after') {
237 return -1;
238 }
239 }
240 return comparator.call(this, a, b);
241 };
242 }
243 var Facet = Facets.Facet = Backbone.Model.extend({
244 initialize: function(data, options) {
245 this.set('all', true);
246 (function(self) {
247 var skipCallback;
248 self.on('item-change', function() {
249 if (skipCallback === 'change:all') {
250 return;
251 }
252 var checkedCount = this.get('items').countBy(function(facetItem) {
253 return facetItem.get('checked');
254 }).true;
255 skipCallback = 'item-change';
256 try {
257 this.set('all', checkedCount === 0);
258 } finally {
259 skipCallback = null;
260 }
261 }, self);
262 self.on('change:all', function() {
263 if (skipCallback === 'item-change') {
264 return;
265 }
266 skipCallback = 'change:all';
267 try {
268 this.get('items').invoke('set', 'checked', false);
269 } finally {
270 skipCallback = null;
271 }
272 }, self);
273 })(this);
274 var sortKeyExtractor = options.sortKeyExtractor;
275 if (!sortKeyExtractor) {
276 sortKeyExtractor = getItemKeyAccessor;
277 }
278 var comparator = options.comparator;
279 if (!comparator) {
280 comparator = Sort.standard;
281 } else if (typeof comparator === 'string') {
282 comparator = Sort[comparator];
283 }
284 switch (options.facetType) {
285 case 'year':
286 case 'range':
287 comparator = wrapComparatorForAllBeforeAfter(comparator, ['all', 'before'].indexOf(options.other) !== -1, ['all', 'after'].indexOf(options.other) !== -1);
288 break;
289 }
290 var formatter = options.formatter;
291 if (!formatter) {
292 var labelMap = options.labelMap;
293 if (labelMap) {
294 formatter = function(item) {
295 var value = item.get('key');
296 var label = options.labelMap[value.toUpperCase()];
297 return label ? label : value;
298 };
299 } else {
300 formatter = getItemKeyAccessor;
301 }
302 }
303 this.formatter = formatter;
304 comparator = (function(comparator) {
305 return function facetItemValue(a, b) {
306 var keyA = sortKeyExtractor(a);
307 var keyB = sortKeyExtractor(b);
308 switch (this.get('orderByDirection')) {
309 case 'desc':
310 var tmp = keyA;
311 keyA = keyB;
312 keyB = tmp;
313 break;
314 }
315 return comparator.call(this, sortKeyExtractor(a), sortKeyExtractor(b));
316 };
317 })(comparator);
318 this.set('items', new ItemCollection([], {comparator: comparator}));
319 this.facetType = options.facetType;
320 this.other = options.other;
321 this.queryField = options.queryField;
322 this.queryValue = options.queryValue;
323 this.bins = options.bins;
324 switch (this.facetType) {
325 case 'year':
326 this.rangeStart = options.start;
327 this.rangeEnd = options.end;
328 this.rangeGap = '+' + (options.gap ? options.gap : 1) + 'YEAR';
329 break;
330 case 'range':
331 this.rangeStart = options.start;
332 this.rangeEnd = options.end;
333 this.rangeGap = options.gap;
334 break;
335 case 'field':
336 break;
337 default:
338 var e = new Error('Unsupported facet type: ' + this.facetType);
339 e.facet = this;
340 throw e;
341 }
342 return Facet.__super__.initialize.apply(this, arguments);
343 }
344 });
345 var ItemCollection = Backbone.Collection.extend({
346 remove: function() {
347 }
348 });
349 var Item = Facet.Item = Backbone.Model.extend({
350 idAttribute: 'key',
351 defaults: {
352 checked: false,
353 hidden: false,
354 },
355 _initialize: function(data, options) {
356 this.on('add', function(model, parent, options) {
357 // added to a collection
358 this.on('change:checked', function() {
359 parent.trigger('item-change');
360 });
361 }, this);
362 return Item.__super__.initialize.apply(this, arguments);
363 }
364 });
365
366 return Facets;
367 })();
368 var Pagination = (function() {
369 function stepFalse(e) {
370 e.preventDefault();
371 return false;
372 }
373 var Pagination = Backbone.Model.extend({
374 defaults: {
375 currentPage: 1,
376 hasNext: false,
377 hasNextJumpPage: false,
378 hasPrevious: false,
379 hasPreviousJumpPage: false,
380 nextJumpPage: undefined,
381 nextPage: undefined,
382 pages: [],
383 pageJump: 5,
384 pageSize: 10,
385 previousJumpPage: undefined,
386 previousPage: undefined,
387 totalCount: 0,
388 totalPages: 0,
389 },
390 initialize: function(data, options) {
391 var buildPages = function buildPages() {
392 var currentPage = parseInt(this.get('currentPage'));
393 var pageSize = parseInt(this.get('pageSize'));
394 var totalCount = parseInt(this.get('totalCount'));
395 if (!currentPage || !pageSize || !totalCount) {
396 return;
397 }
398 var pages = [];
399 var totalPages = Math.floor((totalCount + pageSize - 1) / pageSize);
400 function addPage(self, i) {
401 pages.push({
402 current: i === currentPage,
403 jump: function() {
404 self.set('currentPage', i);
405 return true;
406 },
407 number: i,
408 });
409 }
410 var startAt = currentPage - 4, endAt = currentPage + 5;
411 if (startAt < 1) {
412 endAt += (1 - startAt);
413 }
414 if (endAt > totalPages) {
415 startAt -= endAt - totalPages - 1;
416 endAt = totalPages + 1;
417 }
418 if (startAt < 1) {
419 startAt = 1;
420 }
421 if (endAt - startAt < 9) {
422 /* global console: false */
423 console.log('foo');
424 }
425
426 for (var i = startAt; i < endAt; i++) {
427 if (i > 0 && i <= totalPages) {
428 addPage(this, i);
429 }
430 }
431 var hasPrevious = currentPage > 1;
432 var hasNext = currentPage < totalPages;
433 var pageJump = this.get('pageJump');
434 var nextJumpPage, previousJumpPage, hasNextJump, hasPreviousJump;
435 if (pageJump) {
436 nextJumpPage = currentPage + pageJump;
437 previousJumpPage = currentPage - pageJump;
438 hasNextJump = nextJumpPage < totalPages;
439 hasPreviousJump = previousJumpPage > 0;
440 } else {
441 hasNextJump = false;
442 hasPreviousJump = false;
443 }
444 this.set({
445 hasNext: hasNext,
446 hasNextJump: hasNextJump,
447 hasPrevious: hasPrevious,
448 hasPreviousJump: hasPreviousJump,
449 nextJumpPage: hasNextJump ? nextJumpPage : undefined,
450 nextPage: hasNext ? currentPage + 1 : undefined,
451 pages: pages,
452 previousJumpPage: hasNextJump ? previousJumpPage : undefined,
453 previousPage: hasPrevious ? currentPage - 1 : undefined,
454 totalPages: totalPages,
455 });
456 };
457 this.on('change:pageSize change:currentPage change:totalCount', buildPages, this);
458 var installActions = _.bind(function installActions(eventName, nextName, hasNextName, previousName, hasPreviousName, pageCount) {
459 this.on(eventName, function() {
460 var hasNext = this.get(hasNextName);
461 var hasPrevious = this.get(hasPreviousName);
462 var next, previous;
463 if (hasNext) {
464 next = _.bind(function(e) {
465 e.preventDefault();
466 this.set('currentPage', this.get('currentPage') + pageCount);
467 return true;
468 }, this);
469 } else {
470 next = stepFalse;
471 }
472 if (hasPrevious) {
473 previous = _.bind(function(e) {
474 e.preventDefault();
475 this.set('currentPage', this.get('currentPage') - pageCount);
476 return true;
477 }, this);
478 } else {
479 previous = stepFalse;
480 }
481 this.set(nextName, next);
482 this.set(previousName, previous);
483 }, this);
484 this[nextName] = _.bind(function() {
485 return this.get(nextName)();
486 }, this);
487 this[previousName] = _.bind(function() {
488 return this.get(previousName)();
489 }, this);
490 }, this);
491 this.on('change:pageJump', function() {
492 var pageJump = this.get('pageJump');
493 installActions('change:hasNextJump change:hasPreviousJump', 'nextJump', 'hasNextJump', 'previousJump', 'hasPreviousJump', pageJump);
494 }, this);
495 installActions('change:hasNext change:hasPrevious', 'next', 'hasNext', 'previous', 'hasPrevious', 1);
496 buildPages.apply(this);
497 this.trigger('change:pageJump');
498 return Pagination.__super__.initialize.apply(this, arguments);
499 }
500 });
501 return Pagination;
502 })();
503 var Ordering = Backbone.Model.extend({
504 defaults: {
505 value: null,
506 items: new Backbone.Collection(),
507 },
508 parse: function(data) {
509 var result = _.clone(data);
510 if (result.items && !(result.items instanceof Backbone.Collection)) {
511 result.items = new Backbone.Collection(result.items, {parse: true});
512 }
513 return result;
514 },
515 });
516 var SolrSearch = Pagination.extend({
517 url: function url() {
518 return this.constructor.selectUrl;
519 },
520 defaults: function defaults() {
521 return _.extend(_.result(SolrSearch.__super__, 'defaults'), {
522 initializing: true,
523 initialized: false,
524 query: '',
525 results: new Backbone.Collection(),
526 ordering: new Ordering({items: this.constructor.orderingItems}, {parse: true}),
527 facets: new SolrFacets(this.constructor.facets),
528 });
529 },
530 initialize: function(data, options) {
531 console.log('SolrSearch:initialize');
532 options = (options || {});
533 this.set('options', new Backbone.Model({
534 faceting: !!options.faceting,
535 highlighting: !!options.highlighting,
536 }));
537 this._resetItems = {};
538 this._doSearch = _.debounce(_.bind(function() {
539 console.log('doSearch[b], about to call fetch');
540 this.fetch({
541 data: this.toJSON(),
542 merge: false,
543 traditional: true,
544 });
545 }, this), 250);
546 this._doSearch = (function(doSearch) {
547 return function() {
548 console.log('doSearch[a]', this._skipSearch);
549 if (!this._skipSearch) {
550 doSearch();
551 }
552 };
553 })(this._doSearch);
554
555 this.get('options').on('change:faceting change:highlighting', this._doSearch);
556 this.on('change:query', function() {
557 // this is silent, which causes the change events to not fire;
558 // this is ok, because the next .on will also see the change
559 // event on query, and resend the search
560 this._resetItems.currentPage = true;
561 this._resetItems.facets = true;
562 this._resetItems.ordering = true;
563 this._doSearch();
564 }, this);
565 this.on('change:pageSize', function() {
566 this._resetItems.currentPage = true;
567 this._doSearch();
568 }, this);
569 this.on('change:currentPage', this._doSearch);
570 this.get('facets').on('item-change', function() {
571 this._resetItems.currentPage = true;
572 this._doSearch();
573 }, this);
574 this.get('ordering').on('change:value', function() {
575 this._resetItems.currentPage = true;
576 this._doSearch();
577 }, this);
578 return SolrSearch.__super__.initialize.apply(this, arguments);
579 },
580 parse: function parse(data, options) {
581 // console.debug('SEARCH', data);
582 this._skipSearch = true;
583 try {
584 var resetItems = this._resetItems;
585 if (resetItems.currentPage) {
586 this.set('currentPage', 1);
587 }
588 if (resetItems.facets) {
589 this.get('facets').resetSearch();
590 }
591 if (resetItems.ordering) {
592 this.get('ordering').set('value', this.get('ordering').get('items').at(0).get('value'));
593 }
594 } finally {
595 delete this._skipSearch;
596 }
597 this._resetItems = {};
598 var facets = this.get('facets');
599 if (this.get('options').get('faceting')) {
600 facets.applyFacetResults(data);
601 console.log('facets', facets);
602 } else {
603 facets.resetSearch();
604 }
605 var list = [];
606 var highlighting = this.get('options').get('highlighting') ? data.highlighting : {};
607
608 var self = this;
609 _.each(data.response.docs, function(doc, index) {
610 var itemHighlighting = highlighting[doc.id];
611 if (!itemHighlighting) {
612 itemHighlighting = {};
613 }
614 list.push(self.searchParseDoc(doc, index, itemHighlighting));
615 });
616
617 return {
618 initializing: false,
619 initialized: true,
620 results: new Backbone.Collection(list),
621 facets: facets,
622 options: this.get('options'),
623 totalCount: data.response.numFound,
624 queryTime: (data.responseHeader.QTime / 1000).toFixed(2),
625 hasResults: list.length > 0,
626 };
627 },
628 searchParseDoc: function searchParseDoc(doc, index, itemHighlighting) {
629 var fieldsToParse = mergeStaticProps(this.constructor, SolrSearch, {}, 'parsedFieldMap');
630 var result = {};
631 _.each(fieldsToParse, function(value, key) {
632 var parsed;
633 if (_.isFunction(value)) {
634 parsed = value.call(this, doc, index, itemHighlighting);
635 } else if (_.isArray(value)) {
636 parsed = [];
637 _.each(value, function(item, i) {
638 var rawValue = doc[item.name];
639 if (rawValue) {
640 var parsedValue = item.parse.call(this, doc, index, itemHighlighting, rawValue, item.name);
641 if (parsedValue) {
642 parsed.push(parsedValue);
643 }
644 }
645 }, this);
646 } else {
647 parsed = doc[value];
648 }
649 result[key] = parsed;
650 }, this);
651 return result;
652 },
653 toJSON: function() {
654 var currentPage = this.get('currentPage');
655 var pageSize = this.get('pageSize');
656 var sort = this.get('ordering').get('value');
657 var query = this.get('query');
658 query = query.replace(/:+/g, ' ');
659 if (this._resetItems.currentPage) {
660 currentPage = 1;
661 }
662 if (this._resetItems.ordering) {
663 sort = this.get('ordering').get('items').at(0).get('value');
664 }
665 var constructor = this.constructor;
666 var queryFilter = constructor.queryFilter;
667 var result = {
668 defType: 'dismax',
669 qf: queryFilter ? queryFilter : 'text',
670 sort: sort,
671 //'f.sku.hl.fragmenter': 'regex',
672 //'f.sku.hl.mergeContiguous': 'true',
673 //'f.sku.hl.regex.pattern': '[a-zA-Z]+-[0-9]{4}-[0-9]{1}-[a-zA-Z]{1}-[0-9]+',
674 //'f.sku.hl.regex.pattern': '...-....-..-.-....',
675 fl: '*,score',
676 rows: pageSize,
677 start: (currentPage - 1) * pageSize,
678 wt: 'json'
679 };
680 result.fl = mergeStaticProps(this.constructor, SolrSearch, [], 'returnFields');
681 if (this.get('options').get('highlighting')) {
682 result.hl = true;
683 result['hl.fl'] = ['content', 'title'].concat(constructor.extraHighlightFields);
684 }
685 var dateBoostField = constructor.dateBoostField;
686 var popularityBoostField = constructor.popularityBoostField;
687 if (dateBoostField || popularityBoostField) {
688 var boostSum = [];
689 if (dateBoostField) {
690 result.dateBoost = 'recip(ms(NOW,' + dateBoostField + '),3.16e-11,1,1)';
691 boostSum.push('$dateBoost');
692 }
693 if (popularityBoostField) {
694 result.popularityBoost = 'def(' + popularityBoostField + ',0)';
695 boostSum.push('$popularityBoost');
696 }
697 result.qq = query;
698 result.q = '{!boost b=sum(' + boostSum.join(',') + ') v=$qq defType=$defType}';
699 } else {
700 result.q = query;
701 }
702 if (this.get('options').get('faceting')) {
703 var facetFormData = this.get('facets').getFacetFormData();
704 var fq = facetFormData.fq;
705 delete(facetFormData.fq);
706 result = _.extend(result, facetFormData);
707 if (fq.length) {
708 if (result.fq) {
709 result.fq = result.fq.concat(fq);
710 } else {
711 result.fq = fq;
712 }
713 }
714 }
715 return result;
716 },
717 }, {
718 cleanup: function cleanup(txt) {
719 if (txt) {
720 txt = txt.replace(/&amp;/g, '&');
721 txt = txt.replace(/&#39;/g, '\'');
722 txt = txt.replace(/[^a-zA-Z0-9 -\.,:;%<>\/'"|]/g, ' ');
723 }
724 return txt;
725 },
726 returnFields: [
727 'keywords',
728 'last_modified',
729 'path',
730 'score',
731 'title',
732 ],
733 parsedFieldMap: {
734 content: function(doc, index, itemHighlighting) {
735 var content = itemHighlighting.content;
736 if (content && content.constructor === Array) {
737 content = content.join(' ');
738 }
739 if (!content) {
740 content = '';
741 }
742 return this.constructor.cleanup(content);
743 },
744 keywords: 'keywords',
745 lastModified: 'last_modified',
746 path: 'url',
747 score: 'score',
748 title: function(doc, index, itemHighlighting) {
749 var title;
750 if (itemHighlighting.title) {
751 title = itemHighlighting.title.join(' ');
752 } else {
753 title = doc.title[0];
754 }
755 return this.constructor.cleanup(title);
756 },
757 },
758 });
759 SolrSearch.Facets = SolrFacets;
760 SolrSearch.Pagination = Pagination;
761 return SolrSearch;
3 }); 762 });
......