Gitlab Community Edition Instance

Commit 80c65019 authored by mhellka's avatar mhellka
Browse files

Added cdstar-search-proxy plugin.

parent b020fe9e
Pipeline #96102 passed with stage
in 3 minutes and 49 seconds
= Search Proxy
This plugin implements an engine-agnostic search provider by simply forwarding search queries to an external HTTP(S) search gateway. The gateway should accept JSON encoded POST requests as described below, perform the search in the provided user context, and return the results in a format compatible to the standard CDSTAR v3 search API.
User credentials are not forwarded. Instead, the already authenticated user name and group memberships are attached to the search query. To validate forwarded search queries, the whole search body is cryptographically signed with a shared secret key.
== Configuration
class: de.gwdg.cdstar.ext.sproxy.ProxySearchProvider
hmac: Super-Secret-Key
| Pram | Description |
| class | Plugin class name. Always `de.gwdg.cdstar.ext.sproxy.ProxySearchProvider` |
| url | Search gateway URL. This URL must accept POST requests as described below. |
| hmac | Shared secret used to sign the request body for each search request. |
== Search gateway API
Client search requests are forwarded as JSON encoded POST requests to the configured search gateway `url`, which should return a response that is compatible to the standard CDSTAR v3 search API. The JSON payload of the search requests contains multiple fields:
| Field | Type | Description
| vault | String |
Name of the vault this search was issued against.
| query | String |
Search query string.
| limit | Number |
Maximum number of results to return (optional)
| scroll | String |
Scroll ID as returned by a previous search (optional)
| order | List(String) |
List of field names to order by. Individual names may be prefixed with `-` to reverse the order.
| userName | String |
Fully qualified user name. Only defined for authenticated searches.
| userGroups | List(String) |
Groups this user is a member of. Only defined for authenticated searches.
The fields `vault`, `query`, `limit`, `scroll` and `order` correspond to the parameters provided by the searching user. `userName` and `userGrpups` are added by this search provider for successfully authenticated users. To ensure forward compatibility, search gateways should ignore fields they do not know.
<project xmlns="" xmlns:xsi="" xsi:schemaLocation="">
\ No newline at end of file
package de.gwdg.cdstar.ext.proxysearch;
import java.util.List;
import java.util.Set;
class JsonSearchQuery {
public final String vault;
public final String query;
public final int limit;
public final List<String> order;
public final String scroll;
public final String userName;
public final Set<String> userGroups;
JsonSearchQuery(SearchQuery q) {
vault = q.getVault();
query = q.getQuery();
limit = q.getLimit();
order = q.getOrder();
scroll = q.getScrollId();
userName = q.getPrincipal();
userGroups = q.getGroups();
\ No newline at end of file
package de.gwdg.cdstar.ext.proxysearch;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Future;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import com.fasterxml.jackson.core.JsonProcessingException;
import de.gwdg.cdstar.Promise;
import de.gwdg.cdstar.SharedObjectMapper;
import de.gwdg.cdstar.Utils;
import de.gwdg.cdstar.runtime.Config;
import de.gwdg.cdstar.runtime.ConfigException;
import de.gwdg.cdstar.runtime.Plugin;
import de.gwdg.cdstar.runtime.RuntimeContext;
import de.gwdg.cdstar.runtime.listener.RuntimeListener;
import de.gwdg.cdstar.web.common.model.SearchHits;
* This {@link SearchProvider} forwards search requests to an external HTTP(S)
* search gateway as a JSON encoded POST request. Client credentials are not
* forwarded. Instead, the already authenticate principal and its group
* memberships are provided as fields within the POST request to allow
* user-context dependent searches.
* BASIC auth (or a different means of authentication) should be used to
* authenticate the CDSTAR server against the search target. More advanced
* mechanisms (e.g. to prevent a MITM to send arbitrary searches) may be added
* in the future.
public class ProxySearchProvider implements SearchProvider, RuntimeListener {
* TODO: Track in-flight search requests and publish metrics and health state
* TODO: Configure timeouts for the http client.
private final URI target;
private final CloseableHttpAsyncClient http;
private final SecretKeySpec hmacKey;
public ProxySearchProvider(Config cfg) throws ConfigException {
target = cfg.getURI("url");
hmacKey = cfg.hasKey("hmac")
? new SecretKeySpec(cfg.get("hmac").getBytes(StandardCharsets.UTF_8), "HmacSHA256")
: null;
http = HttpAsyncClients.createDefault();
public void onShutdown(RuntimeContext ctx) {
HttpUriRequest buildRequest(SearchQuery q) {
final HttpPost post = new HttpPost(target);
byte[] payload;
try {
payload = SharedObjectMapper.json.writeValueAsBytes(new JsonSearchQuery(q));
} catch (final JsonProcessingException e) {
post.setEntity(new ByteArrayEntity(payload, ContentType.APPLICATION_JSON));
post.setHeader("Accept", "application/json");
if (hmacKey != null) {
try {
final Mac hmac = Mac.getInstance("HmacSHA256");
post.addHeader("X-Body-Signature", "HmacSHA256:" + Utils.bytesToHex(hmac.doFinal(payload)));
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
return post;
public Promise<SearchResult> search(SearchQuery q) {
final Promise<SearchResult> result = Promise.empty();
final HttpUriRequest request = buildRequest(q);
final Future<HttpResponse> future = http.execute(request, new FutureCallback<HttpResponse>() {
public void completed(HttpResponse response) {
if (response.getStatusLine().getStatusCode() != 200) {
result.reject(new IOException("Unexpected response from search backend: " + response));
try {
final SearchHits hits = SharedObjectMapper.json.readValue(response.getEntity().getContent(),
result.resolve(new ProxySearchResult(hits));
} catch (UnsupportedOperationException | IOException e) {
result.reject(new IOException("Unexpected response from search backend", e));
public void failed(Exception ex) {
public void cancelled() {
// Propagate a canceled search request back to the http client
result.then(null, err -> future.cancel(true));
return result;
package de.gwdg.cdstar.ext.proxysearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import de.gwdg.cdstar.web.common.model.SearchHits;
import de.gwdg.cdstar.web.common.model.SearchHits.Hit;
class ProxySearchResult implements SearchResult {
private final List<SearchHit> hits;
private final String scroll;
private final long total;
public ProxySearchResult(SearchHits delegate) {
hits = new ArrayList<>(delegate.hits.size());
scroll = delegate.scroll;
total =;
delegate.hits.forEach(h -> hits.add(new ProxySearchHit(h)));
public List<SearchHit> hits() {
return Collections.unmodifiableList(hits);
public String getScrollID() {
return scroll;
public long getTotal() {
return total;
class ProxySearchHit implements SearchHit {
private final Hit proxyHit;
public ProxySearchHit(Hit hit) {
proxyHit = hit;
public String getId() {
public String getType() {
return proxyHit.type;
public double getScore() {
return proxyHit.score;
public String getName() {
\ No newline at end of file
......@@ -19,6 +19,7 @@
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment