File diff suppressed because it is too large
Load Diff
+456
@@ -0,0 +1,456 @@
|
||||
const express = require('express');
|
||||
const k8s = require('@kubernetes/client-node');
|
||||
const { WebSocketServer } = require('ws');
|
||||
const http = require('http');
|
||||
const path = require('path');
|
||||
const { PassThrough } = require('stream');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use(express.static(path.join(__dirname, 'public')));
|
||||
|
||||
// ── K8s Client Setup ─────────────────────────────────────────────────────────
|
||||
const kc = new k8s.KubeConfig();
|
||||
try {
|
||||
kc.loadFromCluster();
|
||||
console.log('[k8s] Loaded in-cluster config');
|
||||
} catch {
|
||||
kc.loadFromDefault();
|
||||
console.log('[k8s] Loaded default (local) config');
|
||||
}
|
||||
|
||||
const coreApi = kc.makeApiClient(k8s.CoreV1Api);
|
||||
const appsApi = kc.makeApiClient(k8s.AppsV1Api);
|
||||
const customApi = kc.makeApiClient(k8s.CustomObjectsApi);
|
||||
const logger = new k8s.Log(kc);
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
const PATCH_HEADERS = { headers: { 'Content-Type': 'application/strategic-merge-patch+json' } };
|
||||
|
||||
function podRestarts(pod) {
|
||||
return (pod.status?.containerStatuses || [])
|
||||
.reduce((s, c) => s + (c.restartCount || 0), 0);
|
||||
}
|
||||
|
||||
function podPhase(pod) {
|
||||
// Check container states for more granular status
|
||||
const cs = pod.status?.containerStatuses || [];
|
||||
if (cs.some(c => c.state?.waiting?.reason === 'CrashLoopBackOff')) return 'CrashLoop';
|
||||
if (cs.some(c => c.state?.waiting?.reason === 'OOMKilled')) return 'OOMKilled';
|
||||
if (cs.some(c => c.state?.waiting?.reason === 'Error')) return 'Error';
|
||||
return pod.status?.phase || 'Unknown';
|
||||
}
|
||||
|
||||
// ── Routes ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Overview: all deployments with aggregated pod info
|
||||
app.get('/api/overview', async (req, res) => {
|
||||
try {
|
||||
const [depsRes, podsRes, nodesRes] = await Promise.all([
|
||||
appsApi.listDeploymentForAllNamespaces(),
|
||||
coreApi.listPodForAllNamespaces(),
|
||||
coreApi.listNode(),
|
||||
]);
|
||||
|
||||
const pods = podsRes.body.items;
|
||||
const nodes = nodesRes.body.items.map(n => ({
|
||||
name: n.metadata.name,
|
||||
ready: (n.status?.conditions || []).find(c => c.type === 'Ready')?.status === 'True',
|
||||
roles: Object.keys(n.metadata?.labels || {})
|
||||
.filter(k => k.startsWith('node-role.kubernetes.io/'))
|
||||
.map(k => k.replace('node-role.kubernetes.io/', '')),
|
||||
cpu: n.status?.capacity?.cpu,
|
||||
memory: n.status?.capacity?.memory,
|
||||
}));
|
||||
|
||||
const deployments = depsRes.body.items.map(d => {
|
||||
const ns = d.metadata.namespace;
|
||||
const name = d.metadata.name;
|
||||
const sel = d.spec.selector.matchLabels || {};
|
||||
|
||||
const dPods = pods.filter(p => {
|
||||
if (p.metadata.namespace !== ns) return false;
|
||||
const labels = p.metadata.labels || {};
|
||||
return Object.entries(sel).every(([k, v]) => labels[k] === v);
|
||||
});
|
||||
|
||||
return {
|
||||
name,
|
||||
namespace: ns,
|
||||
replicas: d.spec.replicas || 0,
|
||||
readyReplicas: d.status?.readyReplicas || 0,
|
||||
availableReplicas: d.status?.availableReplicas || 0,
|
||||
updatedReplicas: d.status?.updatedReplicas || 0,
|
||||
containers: (d.spec.template.spec.containers || []).map(c => ({
|
||||
name: c.name,
|
||||
image: c.image,
|
||||
imagePullPolicy: c.imagePullPolicy || 'IfNotPresent',
|
||||
})),
|
||||
totalRestarts: dPods.reduce((s, p) => s + podRestarts(p), 0),
|
||||
podCount: dPods.length,
|
||||
conditions: (d.status?.conditions || []).map(c => ({
|
||||
type: c.type, status: c.status, message: c.message,
|
||||
})),
|
||||
createdAt: d.metadata.creationTimestamp,
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ deployments, nodes });
|
||||
} catch (err) {
|
||||
console.error('[overview]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Pods list
|
||||
app.get('/api/pods', async (req, res) => {
|
||||
try {
|
||||
const { namespace } = req.query;
|
||||
const result = namespace && namespace !== 'all'
|
||||
? await coreApi.listNamespacedPod(namespace)
|
||||
: await coreApi.listPodForAllNamespaces();
|
||||
|
||||
const pods = result.body.items.map(p => ({
|
||||
name: p.metadata.name,
|
||||
namespace: p.metadata.namespace,
|
||||
phase: podPhase(p),
|
||||
nodeName: p.spec.nodeName,
|
||||
startTime: p.status?.startTime,
|
||||
ip: p.status?.podIP,
|
||||
containers: (p.status?.containerStatuses || []).map(c => ({
|
||||
name: c.name,
|
||||
ready: c.ready,
|
||||
restartCount: c.restartCount,
|
||||
image: c.image,
|
||||
state: Object.keys(c.state || {})[0] || 'unknown',
|
||||
stateReason: c.state?.waiting?.reason || c.state?.terminated?.reason || null,
|
||||
})),
|
||||
}));
|
||||
|
||||
res.json(pods);
|
||||
} catch (err) {
|
||||
console.error('[pods]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Namespaces
|
||||
app.get('/api/namespaces', async (req, res) => {
|
||||
try {
|
||||
const result = await coreApi.listNamespace();
|
||||
res.json(result.body.items.map(n => n.metadata.name).sort());
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Events / crash history
|
||||
app.get('/api/events', async (req, res) => {
|
||||
try {
|
||||
const { namespace } = req.query;
|
||||
const result = namespace && namespace !== 'all'
|
||||
? await coreApi.listNamespacedEvent(namespace)
|
||||
: await coreApi.listEventForAllNamespaces();
|
||||
|
||||
const CRASH_REASONS = new Set([
|
||||
'BackOff', 'OOMKilling', 'Failed', 'Killing', 'Unhealthy',
|
||||
'CrashLoopBackOff', 'NodeNotReady', 'FailedMount', 'Evicted',
|
||||
'FailedScheduling', 'FailedCreate', 'ImagePullBackOff',
|
||||
]);
|
||||
|
||||
const events = result.body.items
|
||||
.filter(e => e.type === 'Warning' || CRASH_REASONS.has(e.reason))
|
||||
.sort((a, b) => {
|
||||
const ta = new Date(a.lastTimestamp || a.eventTime || 0);
|
||||
const tb = new Date(b.lastTimestamp || b.eventTime || 0);
|
||||
return tb - ta;
|
||||
})
|
||||
.slice(0, 200)
|
||||
.map(e => ({
|
||||
name: e.metadata.name,
|
||||
namespace: e.metadata.namespace,
|
||||
reason: e.reason,
|
||||
message: e.message,
|
||||
type: e.type,
|
||||
count: e.count || 1,
|
||||
firstTime: e.firstTimestamp,
|
||||
lastTime: e.lastTimestamp || e.eventTime,
|
||||
object: {
|
||||
kind: e.involvedObject.kind,
|
||||
name: e.involvedObject.name,
|
||||
namespace: e.involvedObject.namespace,
|
||||
},
|
||||
}));
|
||||
|
||||
res.json(events);
|
||||
} catch (err) {
|
||||
console.error('[events]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Rolling restart a deployment
|
||||
app.post('/api/deployments/:namespace/:name/restart', async (req, res) => {
|
||||
const { namespace, name } = req.params;
|
||||
try {
|
||||
await appsApi.patchNamespacedDeployment(name, namespace, {
|
||||
spec: {
|
||||
template: {
|
||||
metadata: {
|
||||
annotations: { 'kubectl.kubernetes.io/restartedAt': new Date().toISOString() },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, undefined, undefined, undefined, undefined, undefined, PATCH_HEADERS);
|
||||
console.log(`[restart] Deployment ${namespace}/${name}`);
|
||||
res.json({ success: true, message: `Rolling restart triggered for ${name}` });
|
||||
} catch (err) {
|
||||
console.error('[restart]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Delete pod (triggers restart by controller)
|
||||
app.delete('/api/pods/:namespace/:name', async (req, res) => {
|
||||
const { namespace, name } = req.params;
|
||||
try {
|
||||
await coreApi.deleteNamespacedPod(name, namespace);
|
||||
console.log(`[pod-restart] Deleted pod ${namespace}/${name}`);
|
||||
res.json({ success: true, message: `Pod ${name} deleted — controller will restart it` });
|
||||
} catch (err) {
|
||||
console.error('[pod-restart]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Force image update (rolling restart — requires imagePullPolicy: Always)
|
||||
app.post('/api/deployments/:namespace/:name/update', async (req, res) => {
|
||||
const { namespace, name } = req.params;
|
||||
try {
|
||||
await appsApi.patchNamespacedDeployment(name, namespace, {
|
||||
spec: {
|
||||
template: {
|
||||
metadata: {
|
||||
annotations: {
|
||||
'kubectl.kubernetes.io/restartedAt': new Date().toISOString(),
|
||||
'k8s-pilot/forced-update': new Date().toISOString(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, undefined, undefined, undefined, undefined, undefined, PATCH_HEADERS);
|
||||
console.log(`[update] Triggered image update for ${namespace}/${name}`);
|
||||
res.json({ success: true, message: `Image update triggered for ${name}` });
|
||||
} catch (err) {
|
||||
console.error('[update]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Check Docker Hub for newer image digest
|
||||
app.post('/api/deployments/:namespace/:name/check-update', async (req, res) => {
|
||||
const { namespace, name } = req.params;
|
||||
try {
|
||||
const depResult = await appsApi.readNamespacedDeployment(name, namespace);
|
||||
const containers = depResult.body.spec.template.spec.containers;
|
||||
|
||||
const results = await Promise.all(containers.map(async c => {
|
||||
const image = c.image;
|
||||
const colonIdx = image.lastIndexOf(':');
|
||||
const imageName = colonIdx > 0 ? image.substring(0, colonIdx) : image;
|
||||
const tag = colonIdx > 0 ? image.substring(colonIdx + 1) : 'latest';
|
||||
|
||||
// Only Docker Hub images (no external registry prefix)
|
||||
const parts = imageName.split('/');
|
||||
const isDockerHub = parts.length <= 2 && !imageName.includes('.');
|
||||
|
||||
if (!isDockerHub) {
|
||||
return { container: c.name, image, updateCheckSupported: false,
|
||||
reason: 'Only Docker Hub supported for auto-check (GHCR/custom registries: use Force Update)' };
|
||||
}
|
||||
|
||||
const fullName = parts.length === 1 ? `library/${imageName}` : imageName;
|
||||
|
||||
try {
|
||||
const tokenRes = await fetch(
|
||||
`https://auth.docker.io/token?service=registry.docker.io&scope=repository:${fullName}:pull`
|
||||
);
|
||||
const { token } = await tokenRes.json();
|
||||
|
||||
const manifestRes = await fetch(
|
||||
`https://registry-1.docker.io/v2/${fullName}/manifests/${tag}`,
|
||||
{ headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.docker.distribution.manifest.v2+json' } }
|
||||
);
|
||||
|
||||
const remoteDigest = manifestRes.headers.get('docker-content-digest');
|
||||
return { container: c.name, image, tag, remoteDigest, updateCheckSupported: true };
|
||||
} catch (fetchErr) {
|
||||
return { container: c.name, image, updateCheckSupported: false, reason: fetchErr.message };
|
||||
}
|
||||
}));
|
||||
|
||||
res.json(results);
|
||||
} catch (err) {
|
||||
console.error('[check-update]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Trivy VulnerabilityReports (requires Trivy Operator installed)
|
||||
app.get('/api/vulnerabilities', async (req, res) => {
|
||||
const { namespace } = req.query;
|
||||
const GROUP = 'aquasecurity.github.io';
|
||||
const VERSION = 'v1alpha1';
|
||||
const PLURAL = 'vulnerabilityreports';
|
||||
|
||||
try {
|
||||
const result = namespace && namespace !== 'all'
|
||||
? await customApi.listNamespacedCustomObject(GROUP, VERSION, namespace, PLURAL)
|
||||
: await customApi.listClusterCustomObject(GROUP, VERSION, PLURAL);
|
||||
|
||||
const reports = result.body.items.map(r => {
|
||||
const labels = r.metadata?.labels || {};
|
||||
const report = r.report || {};
|
||||
const summary = report.summary || {};
|
||||
return {
|
||||
name: r.metadata.name,
|
||||
namespace: r.metadata.namespace,
|
||||
container: labels['trivy-operator.container-name'],
|
||||
resource: labels['trivy-operator.resource.name'],
|
||||
image: report.artifact
|
||||
? [report.artifact.repository, report.artifact.tag].filter(Boolean).join(':')
|
||||
: null,
|
||||
scannedAt: r.metadata.creationTimestamp,
|
||||
summary: {
|
||||
critical: summary.criticalCount || 0,
|
||||
high: summary.highCount || 0,
|
||||
medium: summary.mediumCount || 0,
|
||||
low: summary.lowCount || 0,
|
||||
none: summary.noneCount || 0,
|
||||
unknown: summary.unknownCount || 0,
|
||||
},
|
||||
vulnerabilities: (report.vulnerabilities || [])
|
||||
.sort((a, b) => {
|
||||
const order = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3, NONE: 4, UNKNOWN: 5 };
|
||||
return (order[a.severity] ?? 9) - (order[b.severity] ?? 9);
|
||||
})
|
||||
.map(v => ({
|
||||
id: v.vulnerabilityID,
|
||||
severity: v.severity,
|
||||
resource: v.resource,
|
||||
installedVersion: v.installedVersion,
|
||||
fixedVersion: v.fixedVersion || '–',
|
||||
title: v.title,
|
||||
primaryLink: v.primaryLink,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ installed: true, reports });
|
||||
} catch (err) {
|
||||
// CRD not found = Trivy Operator not installed
|
||||
const notInstalled =
|
||||
err.statusCode === 404 ||
|
||||
err.body?.reason === 'NotFound' ||
|
||||
(err.message || '').toLowerCase().includes('not found');
|
||||
|
||||
if (notInstalled) return res.json({ installed: false, reports: [] });
|
||||
console.error('[vulnerabilities]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ── SSE: Live log stream ──────────────────────────────────────────────────────
|
||||
app.get('/api/logs/:namespace/:pod/:container', async (req, res) => {
|
||||
const { namespace, pod, container } = req.params;
|
||||
const tail = parseInt(req.query.tail || '200', 10);
|
||||
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('X-Accel-Buffering', 'no'); // disable Nginx buffering
|
||||
res.flushHeaders();
|
||||
|
||||
const logStream = new PassThrough();
|
||||
let request = null;
|
||||
|
||||
logStream.on('data', chunk => {
|
||||
const lines = chunk.toString().split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.trim()) {
|
||||
res.write(`data: ${JSON.stringify({ line })}\n\n`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logStream.on('error', err => {
|
||||
res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
|
||||
res.end();
|
||||
});
|
||||
|
||||
try {
|
||||
request = await logger.log(namespace, pod, container, logStream, {
|
||||
follow: true,
|
||||
tailLines: tail,
|
||||
timestamps: true,
|
||||
pretty: false,
|
||||
});
|
||||
} catch (err) {
|
||||
res.write(`data: ${JSON.stringify({ error: err.message })}\n\n`);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
req.on('close', () => {
|
||||
try { request?.abort?.(); } catch {}
|
||||
logStream.destroy();
|
||||
console.log(`[logs] Stream closed: ${namespace}/${pod}/${container}`);
|
||||
});
|
||||
});
|
||||
|
||||
// ── HTTP + WebSocket Server ───────────────────────────────────────────────────
|
||||
const server = http.createServer(app);
|
||||
const wss = new WebSocketServer({ server, path: '/ws' });
|
||||
|
||||
// Broadcast live cluster status every 8s
|
||||
async function broadcastStatus() {
|
||||
if (wss.clients.size === 0) return;
|
||||
try {
|
||||
const [depsRes, podsRes] = await Promise.all([
|
||||
appsApi.listDeploymentForAllNamespaces(),
|
||||
coreApi.listPodForAllNamespaces(),
|
||||
]);
|
||||
|
||||
const pods = podsRes.body.items;
|
||||
const unhealthy = pods.filter(p => !['Running', 'Succeeded'].includes(p.status?.phase));
|
||||
const crashLoops = pods.filter(p => (p.status?.containerStatuses || [])
|
||||
.some(c => c.state?.waiting?.reason === 'CrashLoopBackOff'));
|
||||
const totalRestarts = pods.reduce((s, p) => s + podRestarts(p), 0);
|
||||
|
||||
const payload = JSON.stringify({
|
||||
type: 'status',
|
||||
deployments: depsRes.body.items.length,
|
||||
pods: pods.length,
|
||||
unhealthy: unhealthy.length,
|
||||
crashLoops: crashLoops.length,
|
||||
totalRestarts,
|
||||
ts: Date.now(),
|
||||
});
|
||||
|
||||
wss.clients.forEach(ws => {
|
||||
if (ws.readyState === 1) ws.send(payload);
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
|
||||
setInterval(broadcastStatus, 8000);
|
||||
|
||||
wss.on('connection', ws => {
|
||||
console.log('[ws] Client connected');
|
||||
broadcastStatus(); // immediate update on connect
|
||||
ws.on('close', () => console.log('[ws] Client disconnected'));
|
||||
});
|
||||
|
||||
const PORT = process.env.PORT || 3000;
|
||||
server.listen(PORT, () => console.log(`🚀 K8s Pilot running on :${PORT}`));
|
||||
Reference in New Issue
Block a user