Browse Source

first draft: make netgraph site out of csv files.

main
Robey 5 years ago
parent
commit
64f14e05d0
  1. 3
      .gitignore
  2. 15
      site/index.html
  3. 16
      site/netgraph.css
  4. 9
      src/arrays.ts
  5. 81
      src/csv_loader.ts
  6. 1329
      src/dygraphs.d.ts
  7. 37
      src/fetch.ts
  8. 109
      src/netgraph.ts
  9. 18
      src/test/test_arrays.ts
  10. 30
      src/test/test_csv_loader.ts
  11. 40
      src/time.ts
  12. 12
      tsconfig.json
  13. 29
      webpack.config.js

3
.gitignore

@ -0,0 +1,3 @@
lib/
node_modules/
site/netgraph.js

15
site/index.html

@ -0,0 +1,15 @@
<!doctype html>
<html>
<head>
<script src="netgraph.js"></script>
<link rel="stylesheet" type="text/css" href="netgraph.css"></link>
</head>
<body>
<h3> Latency to 8.8.8.8 </h3>
<div class="graph-row" csv="ping,ping_droprate">
<div class="graph" timespan="hour"></div>
<div class="graph" timespan="day"></div>
<div class="graph" timespan="week"></div>
</div>
</body>
</html>

16
site/netgraph.css

@ -0,0 +1,16 @@
.graph-row {
display: flex;
flex-direction: row;
margin-top: 1em;
margin-bottom: 1em;
width: 100%;
height: 15em;
}
.graph {
display: flex;
flex: 1 1 auto;
width: 100%;
margin-left: 1em;
margin-right: 1em;
}

9
src/arrays.ts

@ -0,0 +1,9 @@
// groan.
export function range(start: number, end: number, step: number = 1): number[] {
return [...Array(Math.ceil((end - start) / step)).keys()].map(i => i * step + start);
}
export function arrayGrouped<A>(array: A[], groupSize: number): A[][] {
return range(0, array.length, groupSize).map(i => array.slice(i, i + groupSize));
}

81
src/csv_loader.ts

@ -0,0 +1,81 @@
import { arrayGrouped, range } from "./arrays";
import { fetchXHR } from "./fetch";
const BASE_URL = "/collectd/csv";
// how many seconds in each csv file?
const SEGMENT_SIZE = 100000;
// array of rows. each row must have a Date as the first element, tho we can't enforce that in typescript.
export type DygraphRow = Array<Date | number | null>;
export type DygraphData = Array<DygraphRow>;
export async function loadCsvData(name: string, startTime: number, endTime: number): Promise<DygraphData> {
const startSegment = Math.floor(startTime / SEGMENT_SIZE / 1000);
const endSegment = Math.floor(endTime / SEGMENT_SIZE / 1000);
const segments: number[] = [];
for (let i = startSegment; i <= endSegment; i++) segments.push(i);
return Promise.all(segments.map(segment => loadCsv(name, segment))).then(datas => {
return ([] as DygraphData).concat(...datas);
});
}
// load a CSV into dygraph format
export async function loadCsv(name: string, segment: number): Promise<DygraphData> {
try {
const csv = await fetchXHR(`${BASE_URL}/${name}-${segment}.csv`);
const lines = csv.split("\n").filter(line => line.length > 0).map(line => line.trim());
const rv: DygraphData = [];
lines.forEach(line => {
const fields = line.split(",");
if (fields.length == 2) {
rv.push([ new Date(parseFloat(fields[0]) * 1000), parseFloat(fields[1]) ]);
}
});
return rv;
} catch (error) {
return [];
}
}
export function mergeData(...plots: DygraphData[]): DygraphData {
// each incoming plot may have a different row length. compute a "blank
// row" for each one, of all nulls, with the date column stripped. we'll
// use this to add empty rows at the end of the combined plot, if this
// plot ends before the others.
const blanks = plots.map(p => {
if (p.length == 0) return [];
return p[0].slice(1).map(_ => null);
});
const rv: DygraphData = [];
while (true) {
// figure out the smallest timestamp of the plots that still have rows
// left. if all plots are empty, we're done.
const row = plots.map(p => p[0]).filter(x => x !== undefined);
if (row.length == 0) return rv;
const ts = Math.min(...row.map(r => (r[0] as Date).getTime()));
// for each plot:
// - if it's empty, or it's not at the chosen time, use a row of blanks
// - otherwise, take its row and remove it from the plot.
const rows: DygraphRow[] = plots.map((plot, index) => {
if (plot.length == 0) return blanks[index];
if ((plot[0][0] as Date).getTime() != ts) return blanks[index];
return (plot.shift() || []).slice(1);
});
rv.push(([ new Date(ts) ] as DygraphRow).concat(...rows));
}
}
export type Smoosher = (points: Array<Date | number | null>) => (number | null);
// compress rows of the data so that there are no more than maxRows left.
// for each column in a group of smooshed rows, the column values are passed to the smoosher.
export function smooshData(plot: DygraphData, maxRows: number, smoosher: Smoosher): DygraphData {
return arrayGrouped(plot, Math.ceil(plot.length / maxRows)).map(rows => {
return [ rows[0][0] ].concat(range(1, rows[0].length).map(i => smoosher(rows.map(r => r[i]))));
});
}

1329
src/dygraphs.d.ts

File diff suppressed because it is too large

37
src/fetch.ts

@ -0,0 +1,37 @@
const DEFAULT_TIMEOUT = 1000;
let counter = 0;
// really dumb/simple XHR fetch
export function fetchXHR(url: string): Promise<string> {
const requestId = ++counter;
console.log(`XHR ${requestId}: GET ${url}`);
const request = new XMLHttpRequest();
request.open("GET", url, true);
request.timeout = DEFAULT_TIMEOUT;
return new Promise((resolve, reject) => {
request.ontimeout = () => {
console.log(`XHR ${requestId}: Timeout`);
reject(new Error("Timeout"));
};
request.onerror = (e: ErrorEvent) => {
const error = e.error || new Error("Unknown error");
console.log(`XHR ${requestId}: ${error.message}`);
reject(error);
};
request.onload = () => {
console.log(`XHR ${requestId}: ${request.status} ${request.statusText} (${request.responseText.length} bytes)`);
if (request.status >= 400) {
reject(new Error(`${request.status} ${request.statusText}`));
return;
}
resolve(request.responseText);
};
request.send();
});
}

109
src/netgraph.ts

@ -0,0 +1,109 @@
import Dygraph from 'dygraphs';
import { range } from "./arrays";
import { DygraphData, DygraphRow, loadCsvData, mergeData, smooshData } from "./csv_loader";
import { formatDate, DAY, HOUR, WEEK } from "./time";
const MAX_PING = 250;
window.onload = async () => {
const startTime = Date.now() - WEEK;
const endTime = Date.now();
[...document.querySelectorAll(".graph-row")].forEach(async div => {
await makeGraphRow(div as HTMLElement, startTime, endTime);
});
};
async function makeGraphRow(div: HTMLElement, startTime: number, endTime: number): Promise<void> {
const csvNames = (div.getAttribute("csv") || "").split(",");
if (csvNames.length == 0 || csvNames == [ "" ]) {
console.log("! No csv attribute for graph-row:", div);
return;
}
const data = mergeData(...await Promise.all(csvNames.map(name => loadCsvData(name, startTime, endTime))));
[...div.querySelectorAll(".graph")].forEach(async graphDiv => {
const timespan = graphDiv.getAttribute("timespan") || "hour";
let graphStartTime = startTime;
switch (timespan) {
case "week":
graphStartTime = endTime - WEEK;
break;
case "day":
graphStartTime = endTime - DAY;
break;
case "hour":
graphStartTime = endTime - HOUR;
break;
}
const graphData = data.filter(row => {
const ts = row[0] ? (row[0] as Date).getTime() : 0;
return ts >= graphStartTime && ts < endTime;
});
const smooshedData = smooshData(graphData, 500, points => {
const realPoints = points.filter(p => p != null) as number[];
if (realPoints.length == 0) return null;
return realPoints.reduce((a, b) => a + b, 0) / realPoints.length;
});
makeGraph(graphDiv as HTMLElement, smooshedData, csvNames, graphStartTime, endTime);
});
}
function makeGraph(div: HTMLElement, data: DygraphData, names: string[], startTime: number, endTime: number) {
// any column with a name like '_droprate' should be flagged in red if it's ever > 0.
// these are "flag columns" and are not graphed as lines, just red zones.
const RED_BG = toRgba("ff3333", 0.3);
const columnIsFlag = names.map(name => name.match(/_droprate$/));
const flagColumns = columnIsFlag.map((flag, i) => flag ? i : -1).filter(i => i >= 0);
const options: dygraphs.Options = {
legendFormatter: () => "",
animatedZooms: true,
strokeWidth: 1,
highlightCircleSize: 1,
labelsSeparateLines: true,
labels: data[0].map(_ => ""),
dateWindow: [ startTime, endTime ],
valueRange: [ 0, MAX_PING ],
visibility: columnIsFlag.map(flag => !flag),
axes: {
y: {
axisLabelFormatter: n => n + "ms",
valueFormatter: n => "?" + n
},
x: {
axisLabelFormatter: formatDate
}
},
underlayCallback: (canvas, area, g) => {
if (flagColumns.length == 0) return;
range(0, g.numRows()).forEach(row => {
flagColumns.forEach(col => {
// mark a red background on data points with a dropout.
if (g.getValue(row, col + 1) > 0) markError(canvas, area, g, row, RED_BG);
});
});
}
};
new Dygraph(div, data, options);
}
function markError(canvas: CanvasRenderingContext2D, area: dygraphs.Area, g: Dygraph, row: number, color: string) {
const ts = g.getValue(row, 0);
const tsLeft = row == 0 ? ts : g.getValue(row - 1, 0);
const tsRight = row == g.numRows() - 1 ? ts : g.getValue(row + 1, 0);
const left = g.toDomXCoord((ts + tsLeft) / 2);
const right = g.toDomXCoord((ts + tsRight) / 2);
canvas.fillStyle = color;
canvas.fillRect(left, area.y, right - left, area.h);
}
function toRgba(hex: string, alpha: number): string {
const red = parseInt(hex.slice(0, 2), 16);
const green = parseInt(hex.slice(2, 4), 16);
const blue = parseInt(hex.slice(4, 6), 16);
return `rgba(${red}, ${green}, ${blue}, ${alpha})`;
}

18
src/test/test_arrays.ts

@ -0,0 +1,18 @@
import { arrayGrouped, range } from "../arrays";
import "should";
import "source-map-support/register";
describe("arrays", () => {
it("range", () => {
range(3, 6).should.eql([ 3, 4, 5 ]);
range(4, 20, 3).should.eql([4, 7, 10, 13, 16, 19]);
});
it("arrayGrouped", () => {
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 7).should.eql([ [ 1, 2, 3, 4, 5, 6, 7 ] ]);
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 3).should.eql([ [ 1, 2, 3 ], [ 4, 5, 6 ], [ 7 ] ]);
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 2).should.eql([ [ 1, 2 ], [ 3, 4 ], [ 5, 6 ], [ 7 ] ]);
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 1).should.eql([ [ 1 ], [ 2 ], [ 3 ], [ 4 ], [ 5 ], [ 6 ], [ 7 ] ]);
});
});

30
src/test/test_csv_loader.ts

@ -0,0 +1,30 @@
import { DygraphRow, mergeData } from "../csv_loader";
import "should";
import "source-map-support/register";
describe("csv_loader", () => {
it("mergeData", () => {
const plot1: DygraphRow[] = [
[ new Date(100), 1 ],
[ new Date(110), 2 ],
[ new Date(120), 3 ],
];
const plot2: DygraphRow[] = [
[ new Date(105), 50, 51, 52 ],
];
const plot3: DygraphRow[] = [
[ new Date(100), 70, 71 ],
[ new Date(105), 72, 73 ],
[ new Date(110), 74, 75 ]
];
const newPlot = mergeData(plot1, plot2, plot3);
newPlot.should.eql([
[ new Date(100), 1, null, null, null, 70, 71 ],
[ new Date(105), null, 50, 51, 52, 72, 73 ],
[ new Date(110), 2, null, null, null, 74, 75 ],
[ new Date(120), 3, null, null, null, null, null ]
]);
});
});

40
src/time.ts

@ -0,0 +1,40 @@
import Dygraph from 'dygraphs';
export const SECOND = 1000;
export const MINUTE = 60 * SECOND;
export const HOUR = 60 * MINUTE;
export const DAY = 24 * HOUR;
export const WEEK = 7 * DAY;
export function formatDate(date: Date, granularity: number) {
const s = date.toString();
const seconds = s.slice(16, 24); // HH:MM:SS
const hours = s.slice(16, 21); // HH:MM
const weekday = s.slice(0, 3); // Mon
const day = s.slice(8, 10); // 05
const month = s.slice(4, 7); // Jan
const year = s.slice(11, 15); // 2015
switch (granularity) {
case Dygraph.Granularity.SECONDLY:
case Dygraph.Granularity.TWO_SECONDLY:
case Dygraph.Granularity.FIVE_SECONDLY:
case Dygraph.Granularity.TEN_SECONDLY:
case Dygraph.Granularity.THIRTY_SECONDLY:
return seconds;
case Dygraph.Granularity.MINUTELY:
case Dygraph.Granularity.TWO_MINUTELY:
case Dygraph.Granularity.FIVE_MINUTELY:
case Dygraph.Granularity.TEN_MINUTELY:
case Dygraph.Granularity.THIRTY_MINUTELY:
return hours;
case Dygraph.Granularity.HOURLY:
case Dygraph.Granularity.TWO_HOURLY:
case Dygraph.Granularity.SIX_HOURLY:
return hours.slice(0, 2) == "00" ? weekday + "<br>" + day + " " + month : hours;
case Dygraph.Granularity.DAILY:
return weekday + "<br>" + day + " " + month;
default: // WEEKLY, MONTHLY, ...
return day + " " + month + "<br>" + year;
}
}

12
tsconfig.json

@ -0,0 +1,12 @@
{
"compilerOptions": {
"module": "commonjs",
"target": "es6",
"strict": true,
"sourceMap": true,
"noImplicitReturns": true,
"rootDir": "src/",
"outDir": "lib/",
"declaration": false
}
}

29
webpack.config.js

@ -0,0 +1,29 @@
module.exports = {
entry: [ "./src/netgraph.ts" ],
output: {
path: __dirname + "/site",
filename: "netgraph.js",
library: "netgraph"
},
// skip these node-only modules.
externals: [
{ "fs": "null" }
// { "aws-sdk": "null" },
],
node: {
Buffer: false
},
resolve: {
extensions: [ ".ts", ".js", ".handlebars" ],
alias: {
"handlebars": "handlebars/dist/handlebars.js"
}
},
module: {
loaders: [
{ test: /.json$/, loader: "json-loader" },
{ test: /\.ts$/, loader: "ts-loader" },
{ test: /\.handlebars$/, loader: "handlebars-loader" }
]
}
};
Loading…
Cancel
Save