Gitlab Community Edition Instance

Commit e047caa8 authored by mhellka's avatar mhellka
Browse files

Removed (defunct) pool reconfiguration logic.

parent 7d6e8cda
Pipeline #293198 passed with stages
in 12 minutes and 7 seconds
......@@ -5,13 +5,12 @@ import java.util.List;
import de.gwdg.cdstar.config.MapConfig;
import de.gwdg.cdstar.runtime.CDStarRuntime;
import de.gwdg.cdstar.runtime.VaultConfig;
import de.gwdg.cdstar.runtime.services.VaultRegistry;
import picocli.CommandLine.Command;
import picocli.CommandLine.Parameters;
@Command(name = "vault", description = "Manage vaults", subcommands = { VaultCommand.Ls.class,
VaultCommand.Create.class, VaultCommand.Edit.class })
VaultCommand.Create.class })
public class VaultCommand extends AbstractCommand {
@Command(name = "list", description = "List all vaults")
......@@ -55,35 +54,4 @@ public class VaultCommand extends AbstractCommand {
}
}
@Command(name = "edit", description = "Edit an existing vault. Use with caution!")
public static class Edit extends AbstractCommand {
@Parameters(index = "0", description = "Name of the vault.")
String name;
@Parameters(index = "1...", arity = "+", description = "Properties in KEY=VALUE format.")
List<Setting> options = new ArrayList<>();
@Override
protected int run() throws Exception {
try (CDStarRuntime runtime = CDStarRuntime.bootstrap(getMain().getConfig())) {
VaultRegistry vreg = runtime.lookupRequired(VaultRegistry.class);
vreg.discover();
VaultConfig vault = vreg.getVault(name)
.orElseThrow(() -> new IllegalStateException("Vault does not exists: " + name));
MapConfig config = new MapConfig(vault.getPropertyMap());
options.forEach(opt -> config.set(opt.getKey(), opt.getValue()));
if (vault.getPropertyMap().equals(config.toMap())) {
System.out.println("Nohinh changed.");
return 1;
}
vreg.updateVault(name, config);
System.out.println("Vault updated!");
}
return 0;
}
}
}
package de.gwdg.cdstar.runtime.services;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
......@@ -15,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.apache.commons.io.FileExistsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -40,7 +40,7 @@ import de.gwdg.cdstar.runtime.listener.VaultConfigListener;
/*
* TODO: Unload pools with no active sessions after a certain time.
*
*
* TODO: Reconfigure vaults and pools if the Config file changes on-disk.
*/
public class VaultRegistry implements RuntimeListener {
......@@ -48,11 +48,12 @@ public class VaultRegistry implements RuntimeListener {
private static final Logger log = LoggerFactory.getLogger(VaultRegistry.class);
private RuntimeContext runtime;
private final Path dataPath;
private ClassLoader poolClassLoader;
private final Map<String, VaultConfigImpl> vaultConfigs = new ConcurrentHashMap<>();
private RuntimeContext runtime;
private final Map<String, StoragePool> pools = new ConcurrentHashMap<>();
public VaultRegistry(Path dataPath, ClassLoader poolClassLoader) {
this.dataPath = dataPath;
......@@ -81,7 +82,6 @@ public class VaultRegistry implements RuntimeListener {
vaultConfig.toMap());
}
}
}
/**
......@@ -90,184 +90,142 @@ public class VaultRegistry implements RuntimeListener {
public void discover() throws IOException {
try (Stream<Path> ds = Files.list(dataPath)) {
for (final Path vaultDir : Utils.iter(ds)) {
tryLoadVault(vaultDir.getFileName().toString());
var name = vaultDir.getFileName().toString();
if (isValidVaultName(name))
loadVault(name);
}
}
}
@Override
public void onShutdown(RuntimeContext ctx) {
vaultConfigs.forEach((name, vc) -> {
try {
vc.close();
} catch (final Exception e) {
log.warn("Failed to properly close vault: {}", vc.getName(), e);
}
public synchronized void onShutdown(RuntimeContext ctx) {
pools.forEach((name, pool) -> {
Utils.closeQuietly(pool);
});
}
Path getConfigPathFor(String vaultName) {
return dataPath.resolve(vaultName).resolve("vault.yml");
vaultConfigs.clear();
pools.clear();
}
/**
* Try to load a given vault, return true on success and false on any errors. No
* exception is thrown.
* Create a new vault. This can also be called on a non-started runtime.
*/
private synchronized boolean tryLoadVault(String name) {
if (vaultConfigs.containsKey(name)) {
log.debug("Not loading {}. Already loaded", name);
return false;
}
if (!isValidVaultName(name)) {
log.warn("Not loading {}. Not a valid name.", name);
return false;
}
if (!Files.isRegularFile(getConfigPathFor(name))) {
log.info("Not loading {}. No config file", name);
return false;
}
try {
loadVault(name);
return true;
} catch (Exception e) {
log.warn("Failed to load {}", name, e);
return false;
}
public synchronized VaultConfigImpl createVault(String vaultName, Config properties) throws IOException {
if (!isValidVaultName(vaultName))
throw new IllegalArgumentException("Invalid vault name: " + vaultName);
if (vaultConfigs.containsKey(vaultName) || Files.exists(getConfigPathFor(vaultName)))
throw new FileExistsException("Vault config exists: " + getConfigPathFor(vaultName));
var vaultConfig = new VaultConfigImpl(vaultName, properties);
runtime.lookupAll(VaultConfigListener.class).forEach(v -> v.vaultConfigChanged(vaultConfig));
persist(vaultConfig);
vaultConfigs.put(vaultName, vaultConfig);
return vaultConfig;
}
private synchronized VaultConfigImpl loadVault(String vaultName) throws IOException {
public synchronized VaultConfigImpl loadVault(String vaultName) throws IOException {
if (vaultConfigs.containsKey(vaultName))
throw new IllegalStateException("Vault loaded twice: " + vaultName);
return vaultConfigs.get(vaultName);
final Config properties = ConfigLoader.fromFile(getConfigPathFor(vaultName).toFile());
setDefaults(vaultName, properties);
final VaultConfigImpl vci = new VaultConfigImpl(vaultName, properties);
vaultConfigs.put(vci.getName(), vci);
return vci;
}
if (!isValidVaultName(vaultName))
throw new IllegalArgumentException("Invalid vault name");
private synchronized void saveVault(String vaultName, Config properties, boolean reload) throws IOException {
final Path confFile = getConfigPathFor(vaultName);
var confFile = getConfigPathFor(vaultName);
var config = ConfigLoader.fromFile(confFile.toFile());
if (!reload && (vaultConfigs.containsKey(vaultName) || Files.exists(confFile)))
throw new IllegalStateException("Vault exists: " + vaultName);
var vaultConfig = new VaultConfigImpl(vaultName, config);
runtime.lookupAll(VaultConfigListener.class).forEach(v -> v.vaultConfigChanged(vaultConfig));
vaultConfigs.put(vaultName, vaultConfig);
return vaultConfig;
}
setDefaults(vaultName, properties);
synchronized void persist(VaultConfigImpl vault) throws IOException {
final Path confFile = getConfigPathFor(vault.getName());
if (!Files.isDirectory(confFile.getParent()))
Files.createDirectories(confFile.getParent());
ConfigLoader.saveAs(confFile.toFile(), properties);
final VaultConfigImpl vci = new VaultConfigImpl(vaultName, properties);
final VaultConfigImpl old = vaultConfigs.put(vaultName, vci);
runtime.lookupAll(VaultConfigListener.class).forEach(l -> l.vaultConfigChanged(vaultConfigs.get(vaultName)));
if (old != null)
old.close();
ConfigLoader.saveAs(confFile.toFile(), vault.getConfig());
}
private void addMetrics(VaultConfigImpl vci) {
if (vci.pool instanceof NioPool) {
private void addMetrics(String vaultName, StoragePool pool) {
if (pool instanceof NioPool) {
runtime.lookup(MetricRegistry.class).ifPresent(mr -> {
final String name = "pool." + vci.getName();
final NioPool pool = (NioPool) vci.pool;
mr.gauge(name + ".cache.hit", () -> () -> pool.getStats().cacheHitCount());
mr.gauge(name + ".cache.miss", () -> () -> pool.getStats().cacheMissCount());
mr.gauge(name + ".cache.size", () -> () -> pool.getStats().cacheSize());
mr.gauge(name + ".disk.total", () -> () -> pool.getStats().diskTotal());
mr.gauge(name + ".disk.free", () -> () -> pool.getStats().diskFree());
mr.gauge(name + ".sess.count", () -> () -> pool.getStats().sessionCount());
final String name = "pool." + vaultName;
final NioPool nioPool = (NioPool) pool;
mr.gauge(name + ".cache.hit", () -> () -> nioPool.getStats().cacheHitCount());
mr.gauge(name + ".cache.miss", () -> () -> nioPool.getStats().cacheMissCount());
mr.gauge(name + ".cache.size", () -> () -> nioPool.getStats().cacheSize());
mr.gauge(name + ".disk.total", () -> () -> nioPool.getStats().diskTotal());
mr.gauge(name + ".disk.free", () -> () -> nioPool.getStats().diskFree());
mr.gauge(name + ".sess.count", () -> () -> nioPool.getStats().sessionCount());
});
}
}
private void setDefaults(String vaultName, Config properties) {
properties.setDefault("pool.class", NioPool.class.getName());
}
/**
* Create a new vault. This can also be called on a non-started runtime.
*/
public synchronized VaultConfigImpl createVault(String vaultName, Config properties) throws IOException {
saveVault(vaultName, properties, false);
return vaultConfigs.get(vaultName);
private boolean isValidVaultName(String name) {
return vaultNamePattern.matcher(name).matches();
}
/**
* Update and persist vault configuration. Doing this on a started runtime will
* close and re-open associated {@link StoragePool}s.
*/
public VaultConfigImpl updateVault(String vaultName, MapConfig properties) throws IOException {
saveVault(vaultName, properties, true);
return vaultConfigs.get(vaultName);
private Path getConfigPathFor(String vaultName) {
return dataPath.resolve(vaultName).resolve("vault.yml");
}
public StoragePool getPoolFor(VaultConfig vault) {
final VaultConfigImpl vci = (VaultConfigImpl) vault;
final StoragePool pool = vci.pool;
if (pool == null)
return initPool(vci);
return pool;
}
var pool = pools.get(vault.getName());
if (pool != null)
return pool;
private StoragePool initPool(VaultConfigImpl vci) {
synchronized (vci) {
if (vci.pool != null)
return vci.pool;
try {
final Config scoped = new MapConfig(vci.config.with("pool"));
final String poolClassName = scoped.get("class", NioPool.class.getName());
// Resolve relative or missing pool data paths
scoped.set("path", dataPath.resolve(scoped.get("path", vci.getName())).toAbsolutePath().toString());
final Class<?> poolClass = poolClassLoader.loadClass(poolClassName);
if (poolClass == null || !StoragePool.class.isAssignableFrom(poolClass))
throw new ConfigException("Unable to load StoragePool class " + poolClassName);
vci.pool = (StoragePool) poolClass.getConstructor(Config.class).newInstance(scoped);
addMetrics(vci);
return vci.pool;
} catch (final Exception e) {
if (e instanceof PoolError)
throw (PoolError) e;
throw new PoolError("Unable to open vault: " + vci.getName(), e);
}
}
return initPool(vault);
}
private boolean isValidVaultName(String name) {
return vaultNamePattern.matcher(name).matches();
private synchronized StoragePool initPool(VaultConfig vault) {
StoragePool pool = pools.get(vault.getName());
if (pool != null)
return pool;
var config = vaultConfigs.get(vault.getName()).getConfig();
try {
var scopedConfig = new MapConfig(config.with("pool"));
scopedConfig.setDefault("class", NioPool.class.getName());
scopedConfig.set("path", dataPath.resolve(config.get("path", vault.getName())).toAbsolutePath().toString());
var poolClassName = scopedConfig.get("class");
var poolClass = poolClassLoader.loadClass(poolClassName);
if (poolClass == null || !StoragePool.class.isAssignableFrom(poolClass))
throw new ConfigException("Unable to load StoragePool class " + poolClassName);
pool = (StoragePool) poolClass.getConstructor(Config.class).newInstance(scopedConfig);
pools.put(vault.getName(), pool);
addMetrics(vault.getName(), pool);
return pool;
} catch (final Exception e) {
throw new PoolError("Unable to open vault: " + vault.getName(), e);
}
}
public Optional<VaultConfig> getVault(String name) {
// Allow runtime discovery of new vaults
if (!vaultConfigs.containsKey(name))
tryLoadVault(name);
return Optional.ofNullable(vaultConfigs.get(name));
try {
return Optional.of(loadVault(name));
} catch (Exception e) {
log.warn("Failed to load {}", name, e);
return Optional.empty();
}
}
public Set<String> getVaultNames() {
return Collections.unmodifiableSet(vaultConfigs.keySet());
}
public class VaultConfigImpl implements VaultConfig, Closeable {
public class VaultConfigImpl implements VaultConfig {
private StoragePool pool;
private final boolean isPublic;
private final String name;
private final Config config;
public VaultConfigImpl(String name, Config config) {
private VaultConfigImpl(String name, Config config) {
this.name = name;
this.config = config;
isPublic = config.getBool("public");
}
@Override
......@@ -277,7 +235,7 @@ public class VaultRegistry implements RuntimeListener {
@Override
public boolean isPublic() {
return isPublic;
return config.getBool("public");
}
@Override
......@@ -295,14 +253,9 @@ public class VaultRegistry implements RuntimeListener {
return config.toMap();
}
@Override
public synchronized void close() throws IOException {
if (pool == null)
return;
pool.close();
pool = null;
public Config getConfig() {
return config;
}
}
public List<VaultConfig> getAll() {
......
......@@ -49,7 +49,7 @@ public class VaultRegistryTest {
VaultRegistry vr = new VaultRegistry(data.toPath(), this.getClass().getClassLoader());
vr.onInit(runtimeMock);
vr.onStartup(runtimeMock);
VaultConfig cfg = vr.createVault("test", new MapConfig());
vr.createVault("test", new MapConfig());
VaultRegistry vr2 = new VaultRegistry(data.toPath(), this.getClass().getClassLoader());
vr2.onInit(runtimeMock);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment