Gitlab Community Edition Instance

Commit 1a9f15e7 authored by mhellka's avatar mhellka
Browse files

Added tons of JVM metrics

parent ec05b443
Pipeline #115859 passed with stages
in 8 minutes and 38 seconds
......@@ -375,25 +375,48 @@ public class NioPool implements StoragePool {
}
}
/**
* Size (bytes) of the backing file store
*/
public long diskTotal() {
return fsTotal;
}
/**
* Free space (bytes) in the backing file store
*/
public long diskFree() {
return fsFree;
}
/**
* Number of objects currently present in the cache
*/
public long cacheSize() {
return cacheSize;
}
/**
* Total number of cache hits that prevented a disk read
*/
public long cacheHitCount() {
return cacheStats.hitCount();
}
/**
* Total number of cache misses that triggered a disk read
*/
public long cacheMissCount() {
return cacheStats.missCount();
}
/**
* Number of open sessions (transactions)
*/
public int sessionCount() {
return scopes.size();
}
}
}
package de.gwdg.cdstar.runtime.services;
import java.lang.management.ClassLoadingMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.lang.management.ThreadMXBean;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.management.MBeanServer;
import javax.management.MBeanServerConnection;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import com.sun.management.UnixOperatingSystemMXBean;
import de.gwdg.cdstar.runtime.RuntimeContext;
import de.gwdg.cdstar.runtime.listener.RuntimeListener;
import de.gwdg.cdstar.runtime.services.health.HealthService;
import de.gwdg.cdstar.runtime.services.health.HealthStatus;
@SuppressWarnings("restriction")
public class DefaultMetricsAndHealthChecks implements RuntimeListener {
private RuntimeContext runtime;
private HealthService health;
private MetricRegistry metrics;
@Override
public void onInit(RuntimeContext ctx) throws Exception {
......@@ -25,47 +39,123 @@ public class DefaultMetricsAndHealthChecks implements RuntimeListener {
@Override
public void onStartup(RuntimeContext ctx) throws Exception {
final HealthService health = runtime.lookupRequired(HealthService.class);
final MetricRegistry metrics = runtime.lookupRequired(MetricRegistry.class);
final ThreadMXBean tmbean = ManagementFactory.getThreadMXBean();
metrics.gauge("jvm.threads.current", () -> () -> tmbean.getThreadCount());
metrics.gauge("jvm.threads.deamon", () -> () -> tmbean.getDaemonThreadCount());
metrics.gauge("jvm.threads.peak", () -> () -> tmbean.getPeakThreadCount());
metrics.gauge("jvm.threads.total", () -> () -> tmbean.getTotalStartedThreadCount());
final OperatingSystemMXBean osbean = ManagementFactory.getOperatingSystemMXBean();
metrics.gauge("jvm.os.load", () -> () -> osbean.getSystemLoadAverage());
metrics.gauge("jvm.os.cpu", () -> () -> osbean.getAvailableProcessors());
final RuntimeMXBean vmbean = ManagementFactory.getRuntimeMXBean();
metrics.gauge("jvm.vm.uptimeMS", () -> () -> vmbean.getUptime());
final MemoryMXBean mbean = ManagementFactory.getMemoryMXBean();
metrics.gauge("jvm.heap.max", () -> () -> mbean.getHeapMemoryUsage().getMax());
metrics.gauge("jvm.heap.used", () -> () -> mbean.getHeapMemoryUsage().getUsed());
metrics.gauge("jvm.heap.res", () -> () -> mbean.getHeapMemoryUsage().getCommitted());
metrics.gauge("jvm.offheap.max", () -> () -> mbean.getNonHeapMemoryUsage().getMax());
metrics.gauge("jvm.offheap.used", () -> () -> mbean.getNonHeapMemoryUsage().getUsed());
metrics.gauge("jvm.offheap.res", () -> () -> mbean.getNonHeapMemoryUsage().getCommitted());
metrics.gauge("jvm.fincount", () -> () -> mbean.getObjectPendingFinalizationCount());
health = runtime.lookupRequired(HealthService.class);
metrics = runtime.lookupRequired(MetricRegistry.class);
registerOs();
registerVm();
registerMem();
registerPool();
registerThread();
registerClassLoader();
}
private void registerOs() {
final OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
metrics.gauge("jvm.os.load", () -> () -> os.getSystemLoadAverage());
metrics.gauge("jvm.os.cpu", () -> () -> os.getAvailableProcessors());
if (os instanceof UnixOperatingSystemMXBean) {
final UnixOperatingSystemMXBean unix = (UnixOperatingSystemMXBean) os;
metrics.gauge("jvm.os.files.max", () -> () -> unix.getMaxFileDescriptorCount());
metrics.gauge("jvm.os.files.open", () -> () -> unix.getOpenFileDescriptorCount());
metrics.gauge("jvm.os.swap.max", () -> () -> unix.getTotalSwapSpaceSize());
metrics.gauge("jvm.os.swap.free", () -> () -> unix.getFreeSwapSpaceSize());
}
}
private void registerVm() {
final RuntimeMXBean vm = ManagementFactory.getRuntimeMXBean();
metrics.gauge("jvm.uptime", () -> () -> vm.getUptime());
}
private void registerMem() {
final MemoryMXBean mem = ManagementFactory.getMemoryMXBean();
final List<MemoryPoolMXBean> pools = ManagementFactory.getMemoryPoolMXBeans();
metrics.gauge("jvm.heap.max", () -> () -> mem.getHeapMemoryUsage().getMax());
metrics.gauge("jvm.heap.used", () -> () -> mem.getHeapMemoryUsage().getUsed());
metrics.gauge("jvm.heap.committed", () -> () -> mem.getHeapMemoryUsage().getCommitted());
metrics.gauge("jvm.offheap.max", () -> () -> mem.getNonHeapMemoryUsage().getMax());
metrics.gauge("jvm.offheap.used", () -> () -> mem.getNonHeapMemoryUsage().getUsed());
metrics.gauge("jvm.offheap.committed", () -> () -> mem.getNonHeapMemoryUsage().getCommitted());
metrics.gauge("jvm.fincount", () -> () -> mem.getObjectPendingFinalizationCount());
for (final MemoryPoolMXBean pool : pools) {
final String saveName = pool.getName()
.toLowerCase()
.replaceAll("[^a-z0-9]+", " ")
.trim()
.replace(" ", "-")
.replace("-space", "");
final String prefix = "jvm.pool." + saveName;
metrics.gauge(prefix + ".max", () -> () -> pool.getUsage().getMax());
metrics.gauge(prefix + ".used", () -> () -> pool.getUsage().getUsed());
metrics.gauge(prefix + ".committed", () -> () -> pool.getUsage().getCommitted());
if (pool.getCollectionUsage() != null) {
metrics.gauge(prefix + ".used-after-gc", () -> () -> pool.getCollectionUsage().getUsed());
}
}
}
private void registerPool() throws MalformedObjectNameException {
final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
for (final String pool : Arrays.asList("direct", "mapped")) {
final ObjectName on = new ObjectName("java.nio:type=BufferPool,name=" + pool);
metrics.gauge("jvm.buffers." + pool + ".count", () -> new JmxAttributeGauge(mbs, on, "Count"));
metrics.gauge("jvm.buffers." + pool + ".used", () -> new JmxAttributeGauge(mbs, on, "MemoryUsed"));
metrics.gauge("jvm.buffers." + pool + ".capacity", () -> new JmxAttributeGauge(mbs, on, "TotalCapacity"));
}
}
private void registerThread() {
final ThreadMXBean threads = ManagementFactory.getThreadMXBean();
metrics.gauge("jvm.threads.current", () -> () -> threads.getThreadCount());
metrics.gauge("jvm.threads.deamon", () -> () -> threads.getDaemonThreadCount());
metrics.gauge("jvm.threads.peak", () -> () -> threads.getPeakThreadCount());
metrics.gauge("jvm.threads.total", () -> () -> threads.getTotalStartedThreadCount());
health.register("jvm.threads.deadlock", () -> {
final long[] deadlocks = tmbean.findDeadlockedThreads();
final long[] deadlocks = threads.findDeadlockedThreads();
if (deadlocks == null || deadlocks.length == 0)
return HealthStatus.ok();
return HealthStatus.error("Deadlocks detected: " + deadlocks.length);
}).interval(60, TimeUnit.SECONDS);
health.register("jvm.threads.count", () -> {
final int threadcount = tmbean.getThreadCount();
final int threadcount = threads.getThreadCount();
if (threadcount < 128)
return HealthStatus.ok();
if (threadcount < 256)
return HealthStatus.warn("Large number of threads: " + threadcount);
return HealthStatus.error("Large number of threads: " + threadcount);
}).interval(60, TimeUnit.SECONDS);
}
private void registerClassLoader() {
final ClassLoadingMXBean cl = ManagementFactory.getClassLoadingMXBean();
metrics.gauge("jvm.classloader.loaded", () -> () -> cl.getLoadedClassCount());
}
private class JmxAttributeGauge implements Gauge<Object> {
private final MBeanServerConnection mBeanServerConn;
private final ObjectName objectName;
private final String attributeName;
public JmxAttributeGauge(MBeanServerConnection mBeanServerConn, ObjectName objectName, String attributeName) {
this.mBeanServerConn = mBeanServerConn;
this.attributeName = attributeName;
this.objectName = objectName;
}
@Override
public Object getValue() {
try {
return mBeanServerConn.getAttribute(objectName, attributeName);
} catch (final Exception e) {
return null;
}
}
}
}
......@@ -224,6 +224,7 @@ public class VaultRegistry implements RuntimeListener {
mr.gauge(name + ".cache.size", () -> () -> pool.getStats().cacheSize());
mr.gauge(name + ".disk.total", () -> () -> pool.getStats().diskTotal());
mr.gauge(name + ".disk.free", () -> () -> pool.getStats().diskFree());
mr.gauge(name + ".sess.count", () -> () -> pool.getStats().sessionCount());
});
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment