diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java index f444688c3..7a687bef8 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java @@ -23,6 +23,7 @@ package net.yacy.cora.federate.solr.responsewriter; import java.io.IOException; import java.io.Writer; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -49,7 +50,9 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TextField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; public class EnhancedXMLResponseWriter implements QueryResponseWriter { @@ -83,14 +86,14 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { @SuppressWarnings("unchecked") SimpleOrderedMap highlighting = (SimpleOrderedMap) values.get("highlighting"); writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader); - writeDocs(writer, request, response); // this.writeVal("response", response); + writeDocs(writer, request, response, rsp.getReturnFields()); // this.writeVal("response", response); writeProps(writer, "highlighting", highlighting); writer.write(XML_STOP); } public static void write(final Writer writer, final SolrQueryRequest request, final SolrDocumentList sdl) throws IOException { writer.write(XML_START); - writeDocs(writer, request, sdl); + writeDocs(writer, request, sdl, new SolrReturnFields(request)); writer.write(XML_STOP); } @@ -113,7 +116,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { } } - private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response) throws IOException { + private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response, final ReturnFields returnFields) throws IOException { boolean includeScore = false; final int sz = response.size(); writer.write(""); writer.write(lb); } - private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs) throws IOException { + private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs, final ReturnFields returnFields) throws IOException { boolean includeScore = false; final int sz = docs.size(); writer.write(" iterator = docs.iterator(); for (int i = 0; i < sz; i++) { SolrDocument doc = iterator.next(); - writeDoc(writer, doc); + writeDoc(writer, doc, returnFields); } writer.write(""); writer.write(lb); } - private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final List fields, final float score, final boolean includeScore) throws IOException { + private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, + final List fields, final float score, final boolean includeScore, + final ReturnFields returnFields) throws IOException { startTagOpen(writer, "doc", name); if (includeScore) { writeTag(writer, "float", "score", Float.toString(score), false); // this is the special Solr "score" pseudo-field } + + /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter + * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */ + final Map fieldRenamings = returnFields == null ? Collections.emptyMap() + : returnFields.getFieldRenames(); int sz = fields.size(); int fidx1 = 0, fidx2 = 0; @@ -181,30 +191,35 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) { fidx2++; } - SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName); - if (sf == null) { - sf = new SchemaField(fieldName, new TextField()); - } - FieldType type = sf.getType(); - if (fidx1 + 1 == fidx2) { - if (sf.multiValued()) { - startTagOpen(writer, "arr", fieldName); - writer.write(lb); - String sv = value.stringValue(); - writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1); - writer.write(""); - } else { - writeField(writer, type.getTypeName(), value.name(), value.stringValue()); //sf.write(this, f1.name(), f1); - } - } else { - startTagOpen(writer, "arr", fieldName); - writer.write(lb); - for (int i = fidx1; i < fidx2; i++) { - String sv = fields.get(i).stringValue(); - writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i)); - } - writer.write(""); - writer.write(lb); + if(returnFields == null || returnFields.wantsField(fieldName)) { + SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName); + if (sf == null) { + sf = new SchemaField(fieldName, new TextField()); + } + + final String renderedFieldName = fieldRenamings.getOrDefault(fieldName, fieldName); + + FieldType type = sf.getType(); + if (fidx1 + 1 == fidx2) { + if (sf.multiValued()) { + startTagOpen(writer, "arr", renderedFieldName); + writer.write(lb); + String sv = value.stringValue(); + writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1); + writer.write(""); + } else { + writeField(writer, type.getTypeName(), renderedFieldName, value.stringValue()); //sf.write(this, f1.name(), f1); + } + } else { + startTagOpen(writer, "arr", renderedFieldName); + writer.write(lb); + for (int i = fidx1; i < fidx2; i++) { + String sv = fields.get(i).stringValue(); + writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i)); + } + writer.write(""); + writer.write(lb); + } } fidx1 = fidx2; } @@ -233,27 +248,68 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { writer.write(lb); } - public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException { + /** + * Append XML representation of the given Solr document to the writer + * + * @param writer + * an open writer. Must not be null. + * @param doc + * the solr document to write. Must not be null. + * @param returnFields + * the eventual fields return configuration, allowing for example to + * restrict the actually returned fields. May be null. + * @throws IOException + * when a write error occurred + */ + private static final void writeDoc(final Writer writer, final SolrDocument doc, final ReturnFields returnFields) throws IOException { startTagOpen(writer, "doc", null); + + /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter + * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */ + final Map fieldRenamings = returnFields == null ? Collections.emptyMap() + : returnFields.getFieldRenames(); + final Map fields = doc.getFieldValueMap(); for (String key: fields.keySet()) { - if (key == null) continue; + if (key == null) { + continue; + } + if (returnFields != null && !returnFields.wantsField(key)) { + continue; + } Object value = doc.get(key); + + final String renderedFieldName = fieldRenamings.getOrDefault(key, key); + if (value == null) { } else if (value instanceof Collection) { - startTagOpen(writer, "arr", key); + startTagOpen(writer, "arr", renderedFieldName); writer.write(lb); for (Object o: ((Collection) value)) { writeField(writer, null, o); } writer.write(""); writer.write(lb); } else { - writeField(writer, key, value); + writeField(writer, renderedFieldName, value); } } writer.write(""); writer.write(lb); } + + /** + * Append XML representation of the given Solr document to the writer + * + * @param writer + * an open writer. Must not be null. + * @param doc + * the solr document to write. Must not be null. + * @throws IOException + * when a write error occurred + */ + public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException { + writeDoc(writer, doc, null); + } private static void writeField(final Writer writer, final String typeName, final String name, final String value) throws IOException { if (typeName.equals(SolrType.text_general.printName()) || diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java index 679ca5cf5..639685cbb 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java @@ -24,17 +24,13 @@ import java.io.IOException; import java.io.Writer; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.federate.solr.SolrType; -import net.yacy.cora.lod.vocabulary.DublinCore; -import net.yacy.search.schema.CollectionSchema; -import net.yacy.search.schema.WebgraphSchema; - import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexableField; import org.apache.solr.common.params.SolrParams; @@ -50,8 +46,14 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TextField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; +import net.yacy.cora.federate.solr.SolrType; +import net.yacy.cora.lod.vocabulary.DublinCore; +import net.yacy.search.schema.CollectionSchema; +import net.yacy.search.schema.WebgraphSchema; + public class HTMLResponseWriter implements QueryResponseWriter { public static final Pattern dqp = Pattern.compile("\""); @@ -165,7 +167,7 @@ public class HTMLResponseWriter implements QueryResponseWriter { int id = iterator.nextDoc(); Document doc = searcher.doc(id); - LinkedHashMap tdoc = translateDoc(schema, doc); + LinkedHashMap tdoc = translateDoc(schema, doc, rsp.getReturnFields()); String title; @@ -187,19 +189,19 @@ public class HTMLResponseWriter implements QueryResponseWriter { writer.write("
\"API\"\n"); writer.write("This search result can also be retrieved as XML. Click the API icon to see this page as XML.
\n"); - writeDoc(writer, tdoc, title, coreName); + writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields()); while (iterator.hasNext()) { id = iterator.nextDoc(); doc = searcher.doc(id); - tdoc = translateDoc(schema, doc); + tdoc = translateDoc(schema, doc, rsp.getReturnFields()); if(CollectionSchema.CORE_NAME.equals(coreName)) { title = tdoc.get(CollectionSchema.title.getSolrFieldName()); if (title == null) title = ""; } else { title = ""; } - writeDoc(writer, tdoc, title, coreName); + writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields()); } } else { writer.write("No Document Found\n\n"); @@ -209,7 +211,18 @@ public class HTMLResponseWriter implements QueryResponseWriter { writer.write("\n"); } - private static final void writeDoc(final Writer writer, final LinkedHashMap tdoc, final String title, final String coreName) throws IOException { + /** + * Append an html representation of the document fields to the writer. + * @param writer an open output writer. Must not be null. + * @param tdoc the documents fields, mapped from field names to values. Must not be null. + * @param title the document title. May be empty. + * @param coreName the Solr core name. + * @param returnFields the eventual fields return configuration, allowing for example to + * rename fields with a pseudo in the result. May be null. + * @throws IOException when a write error occurred. + */ + private static final void writeDoc(final Writer writer, final LinkedHashMap tdoc, + final String title, final String coreName, final ReturnFields returnFields) throws IOException { writer.write("
\n"); writer.write("
\n"); @@ -225,9 +238,13 @@ public class HTMLResponseWriter implements QueryResponseWriter { } writer.write("
\n"); + /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter + * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */ + final Map fieldRenamings = returnFields == null ? Collections.emptyMap() + : returnFields.getFieldRenames(); for (Map.Entry entry: tdoc.entrySet()) { writer.write("
"); - writer.write(entry.getKey()); + writer.write(fieldRenamings.getOrDefault(entry.getKey(), entry.getKey())); writer.write("
"); if (entry.getKey().equals("sku")) { writer.write("" + entry.getValue() + ""); @@ -241,7 +258,19 @@ public class HTMLResponseWriter implements QueryResponseWriter { writer.write("
\n"); } - public static final LinkedHashMap translateDoc(final IndexSchema schema, final Document doc) { + /** + * Translate an indexed document in a map of field names to field values. + * + * @param schema + * the schema of the indexed document. Must not be null. + * @param doc + * the indexed document. Must not be null. + * @param returnFields + * the eventual fields return configuration, allowing for example to + * restrict the actually returned fields. May be null. + * @return a map of field names to field values + */ + private static final LinkedHashMap translateDoc(final IndexSchema schema, final Document doc, final ReturnFields returnFields) { List fields = doc.getFields(); int sz = fields.size(); int fidx1 = 0, fidx2 = 0; @@ -253,29 +282,43 @@ public class HTMLResponseWriter implements QueryResponseWriter { while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) { fidx2++; } - SchemaField sf = schema.getFieldOrNull(fieldName); - if (sf == null) sf = new SchemaField(fieldName, new TextField()); - FieldType type = sf.getType(); + if(returnFields == null || returnFields.wantsField(fieldName)) { + SchemaField sf = schema.getFieldOrNull(fieldName); + if (sf == null) { + sf = new SchemaField(fieldName, new TextField()); + } + FieldType type = sf.getType(); - if (fidx1 + 1 == fidx2) { - if (sf.multiValued()) { - String sv = value.stringValue(); - kv.put(fieldName, field2string(type, sv)); - } else { - kv.put(fieldName, field2string(type, value.stringValue())); - } - } else { - int c = 0; - for (int i = fidx1; i < fidx2; i++) { - String sv = fields.get(i).stringValue(); - kv.put(fieldName + "_" + c++, field2string(type, sv)); - } + if (fidx1 + 1 == fidx2) { + if (sf.multiValued()) { + String sv = value.stringValue(); + kv.put(fieldName, field2string(type, sv)); + } else { + kv.put(fieldName, field2string(type, value.stringValue())); + } + } else { + int c = 0; + for (int i = fidx1; i < fidx2; i++) { + String sv = fields.get(i).stringValue(); + kv.put(fieldName + "_" + c++, field2string(type, sv)); + } + } } fidx1 = fidx2; } return kv; } + + /** + * Translate an indexed document in a map of field names to field values. + * @param schema the schema of the indexed document. Must not be null. + * @param doc the indexed document. Must not be null. + * @return a map of field names to field values + */ + public static final LinkedHashMap translateDoc(final IndexSchema schema, final Document doc) { + return translateDoc(schema, doc, null); + } private static String field2string(final FieldType type, final String value) { String typeName = type.getTypeName(); diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java index 497eb1236..7371816a2 100644 --- a/source/net/yacy/http/servlets/SolrSelectServlet.java +++ b/source/net/yacy/http/servlets/SolrSelectServlet.java @@ -99,8 +99,7 @@ public class SolrSelectServlet extends HttpServlet { RESPONSE_WRITER.putAll(SolrCore.DEFAULT_RESPONSE_WRITERS); XSLTResponseWriter xsltWriter = new XSLTResponseWriter(); OpensearchResponseWriter opensearchResponseWriter = new OpensearchResponseWriter(); - @SuppressWarnings("rawtypes") - NamedList initArgs = new NamedList(); + NamedList initArgs = new NamedList<>(); xsltWriter.init(initArgs); RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());