11 changed files with 3888 additions and 3781 deletions
File diff suppressed because it is too large
@ -1,9 +0,0 @@ |
|||
// groan.
|
|||
|
|||
export function range(start: number, end: number, step: number = 1): number[] { |
|||
return [...Array(Math.ceil((end - start) / step)).keys()].map(i => i * step + start); |
|||
} |
|||
|
|||
export function arrayGrouped<A>(array: A[], groupSize: number): A[][] { |
|||
return range(0, array.length, groupSize).map(i => array.slice(i, i + groupSize)); |
|||
} |
|||
@ -1,284 +0,0 @@ |
|||
import { range } from "./arrays"; |
|||
|
|||
const COOKIE_FLOAT = 8.642135e+130; |
|||
|
|||
/* |
|||
* format of an RRD file: |
|||
* - header |
|||
* - magic: u8[4] -- "RRD\0" |
|||
* - version: u8[5] -- why 5?! don't question it! |
|||
* - cookie: f64 -- ✨ aligned ✨, so at offset 12 or 16 |
|||
* - data_source_count: int |
|||
* - rra_count: int |
|||
* - pdp_step: int -- seconds |
|||
* - unused: f64[10] |
|||
* - data_series headers: (x data_source_count) |
|||
* - name: u8[20] |
|||
* - type: u8[20] |
|||
* - options: f64[10] -- we don't care about any of them |
|||
* - RRA headers: (x rra_count) |
|||
* - name: u8[20] |
|||
* - row_count: int |
|||
* - pdp_count: int -- how many PDPs per value |
|||
* - options: f64[10] -- we don't care about any of them |
|||
* - update header |
|||
* - last_sec: int -- time last file was updated |
|||
* - last_usec: int |
|||
* - data_series configs: (x data_source_count) |
|||
* - last_data: u8[30] |
|||
* - scratch: f64[10] |
|||
* - RRA per data_series configs: (x data_source_count x rra_count) |
|||
* - scratch: f64[10] |
|||
* - current row per RRA: (x rra_count) |
|||
* - row_index: int |
|||
* - actual data |
|||
* - data: f64[row_count][rra_count] |
|||
* |
|||
* notes: |
|||
* - the int fields can be u32 or u64, ✨ you figure it out ✨ |
|||
* - the f64 cookie should tell us which endian we're dealing with |
|||
* - RRA = "round robin archive" |
|||
* - PDP = "primary data point" (vs CDP "consolidated data point") |
|||
* - the data is in grids, one per RRA, each row being f64 x ds_count |
|||
* - in theory, data points could be ints? the file format bends over |
|||
* backward to allow this, but everyone seems to assume all data is f64. |
|||
*/ |
|||
|
|||
export type Row = [ number, number | null ]; |
|||
export type Table = Row[]; |
|||
|
|||
export interface DataSource { |
|||
index: number; |
|||
name: string; |
|||
type: string; |
|||
} |
|||
|
|||
export interface Archive { |
|||
index: number; |
|||
name: string; |
|||
rows: number; |
|||
stepCount: number; |
|||
currentRow: number; |
|||
|
|||
// computed:
|
|||
offset: number; |
|||
startTime: number; |
|||
} |
|||
|
|||
export class RrdFile { |
|||
offset: number = 0; |
|||
|
|||
version: string = ""; |
|||
littleEndian = true; |
|||
|
|||
floatAlign: number = 0; |
|||
intAlign: number = 0; |
|||
intIs64: boolean = false; |
|||
|
|||
dsCount: number = 0; |
|||
archiveCount: number = 0; |
|||
pdpStep: number = 0; |
|||
lastUpdate: number = 0; |
|||
|
|||
dataSources: DataSource[] = []; |
|||
archives: Archive[] = []; |
|||
|
|||
constructor(public data: DataView) { |
|||
this.validate(); |
|||
this.readHeader(); |
|||
this.parse(); |
|||
} |
|||
|
|||
getKeys(): string[] { |
|||
return ([] as string[]).concat(...this.dataSources.map(ds => { |
|||
return this.archives.filter(a => a.stepCount == 1).map(a => { |
|||
return `${ds.name}:${a.name}`; |
|||
}); |
|||
})); |
|||
} |
|||
|
|||
toString(): string { |
|||
const version = parseInt(this.version, 10); |
|||
const endian = this.littleEndian ? "LE" : "BE"; |
|||
const iSize = this.intIs64 ? 64 : 32; |
|||
|
|||
return `RrdFile(${endian}, version=${version}, sources=${this.dsCount}, archives=${this.archiveCount}, ` + |
|||
`lastUpdate=${this.lastUpdate}, step=${this.pdpStep}, ` + |
|||
`int=${iSize}:${this.intAlign}, float=64:${this.floatAlign})`; |
|||
} |
|||
|
|||
validate() { |
|||
if (this.data.byteLength < 16) throw new Error("Truncated RRD"); |
|||
this.offset = 0; |
|||
if (this.readAscii(4) != "RRD") throw new Error("Not an RRD file"); |
|||
this.version = this.readAscii(5); |
|||
if ([ "0001", "0003", "0004" ].indexOf(this.version) < 0) { |
|||
throw new Error(`Unknown RRD version ${this.version}`); |
|||
} |
|||
} |
|||
|
|||
readHeader() { |
|||
// read magic float
|
|||
if (this.data.getUint32(12, this.littleEndian) == 0) { |
|||
// 64-bit alignment
|
|||
this.floatAlign = 8; |
|||
this.offset = 16; |
|||
if (this.data.getFloat64(this.offset, true) == COOKIE_FLOAT) { |
|||
this.littleEndian = true; |
|||
} else if (this.data.getFloat64(this.offset, false) == COOKIE_FLOAT) { |
|||
this.littleEndian = false; |
|||
} else { |
|||
throw new Error("RRD is missing magic float"); |
|||
} |
|||
|
|||
if (this.data.getInt32(this.offset + 8 + 4, this.littleEndian) == 0) { |
|||
this.intAlign = 8; |
|||
this.intIs64 = true; |
|||
} else { |
|||
this.intAlign = 4; |
|||
this.intIs64 = false; |
|||
} |
|||
} else { |
|||
this.floatAlign = 4; |
|||
this.offset = 12; |
|||
this.intAlign = 4; |
|||
this.intIs64 = false; |
|||
if (this.data.getFloat64(this.offset, true) == COOKIE_FLOAT) { |
|||
this.littleEndian = true; |
|||
} else if (this.data.getFloat64(this.offset, false) == COOKIE_FLOAT) { |
|||
this.littleEndian = false; |
|||
} else { |
|||
throw new Error("RRD is missing magic float"); |
|||
} |
|||
} |
|||
|
|||
this.offset += 8; |
|||
this.dsCount = this.readInt(); |
|||
this.archiveCount = this.readInt(); |
|||
this.pdpStep = this.readInt(); |
|||
// skip 10x f64:
|
|||
this.alignFloat(); |
|||
this.offset += 10 * 8; |
|||
|
|||
if (this.dsCount == 0 || this.archiveCount == 0 || this.pdpStep == 0) throw new Error("RRD file is corrupted"); |
|||
} |
|||
|
|||
parse() { |
|||
this.dataSources = range(0, this.dsCount).map(i => { |
|||
const name = this.readAscii(20); |
|||
const type = this.readAscii(20); |
|||
this.alignFloat(); |
|||
this.offset += 10 * 8; |
|||
return { index: i, name, type }; |
|||
}); |
|||
this.archives = range(0, this.archiveCount).map(i => { |
|||
const name = this.readAscii(20); |
|||
const rows = this.readInt(); |
|||
const stepCount = this.readInt(); |
|||
this.alignFloat(); |
|||
this.offset += 10 * 8; |
|||
return { index: i, name, rows, stepCount, currentRow: 0, offset: -1, startTime: -1 }; |
|||
}); |
|||
|
|||
this.lastUpdate = this.readInt(); |
|||
this.lastUpdate += (this.readInt() / Math.pow(10, 6)); |
|||
// so, uh, really, realistically, this is clamped to the step count.
|
|||
this.lastUpdate = Math.floor(this.lastUpdate / this.pdpStep) * this.pdpStep; |
|||
|
|||
// skip un-interesting configs.
|
|||
// (30 moves to 32 at both 4 & 8 alignment, so we can skip alignment checks.)
|
|||
this.offset += this.dsCount * (32 + 8 * 10); |
|||
this.offset += this.dsCount * this.archiveCount * 8 * 10; |
|||
|
|||
this.archives.forEach(archive => { |
|||
archive.currentRow = this.readInt(); |
|||
}); |
|||
|
|||
// everything else in the file is data tables. calculate each archive's offset.
|
|||
this.archives.forEach(a => { |
|||
a.offset = this.offset; |
|||
this.offset += a.rows * 8 * this.dsCount; |
|||
a.startTime = this.lastUpdate - a.stepCount * this.pdpStep * (a.rows - 1); |
|||
}); |
|||
} |
|||
|
|||
getSeries(key: string, startTime: number, endTime: number): Table { |
|||
const [ dsName, archiveName ] = key.split(":"); |
|||
|
|||
const ds = this.dataSources.filter(d => d.name == dsName)[0]; |
|||
// sort matching archives with finest granularity up front.
|
|||
const archives = this.archives.filter(a => a.name == archiveName).sort((a, b) => b.startTime - a.startTime); |
|||
if (ds == null || archives.length == 0) return []; |
|||
|
|||
// pick the archive with with the start time that covers our range (if any).
|
|||
const archive = archives.filter(a => a.startTime < startTime)[0] || archives[archives.length - 1]; |
|||
const step = this.pdpStep * archive.stepCount; |
|||
|
|||
// clamp start time and end time to actual db values.
|
|||
if (startTime < archive.startTime) startTime = archive.startTime; |
|||
if (endTime > this.lastUpdate) endTime = this.lastUpdate + step; |
|||
|
|||
// make sure both start & end times are clamped to the steps used by this archive
|
|||
const startDist = this.lastUpdate - startTime; |
|||
startTime = this.lastUpdate - Math.floor(startDist / step) * step; |
|||
const endDist = this.lastUpdate - endTime; |
|||
endTime = this.lastUpdate - Math.ceil(endDist / step) * step; |
|||
|
|||
let firstRow = (startTime - archive.startTime) / step + archive.currentRow + 1; |
|||
if (firstRow >= archive.rows) firstRow -= archive.rows; |
|||
const stride = this.dsCount * 8; |
|||
return range(startTime, endTime, step).map((ts, i) => { |
|||
if (firstRow + i >= archive.rows) firstRow -= archive.rows; |
|||
const offset = archive.offset + (firstRow + i) * stride; |
|||
const d = this.data.getFloat64(offset, this.littleEndian); |
|||
return [ ts, isNaN(d) ? null : d ] as [ number, number | null ]; |
|||
}); |
|||
} |
|||
|
|||
private readAscii(length: number): string { |
|||
let rv = ""; |
|||
let start = this.offset; |
|||
const end = this.offset + length; |
|||
while (end > start && this.data.getUint8(start) != 0) { |
|||
rv += String.fromCharCode(this.data.getUint8(start)); |
|||
start++; |
|||
} |
|||
this.offset += length; |
|||
return rv; |
|||
} |
|||
|
|||
private readInt(): number { |
|||
this.alignInt(); |
|||
if (this.intIs64) { |
|||
// we only ever use 32 bits. :/
|
|||
const rv = this.littleEndian ? |
|||
this.data.getUint32(this.offset, true) : |
|||
this.data.getUint32(this.offset + 4, false); |
|||
this.offset += 8; |
|||
return rv; |
|||
} |
|||
const rv = this.data.getUint32(this.offset, this.littleEndian); |
|||
this.offset += 4; |
|||
return rv; |
|||
} |
|||
|
|||
private readFloat(): number { |
|||
this.alignFloat(); |
|||
const rv = this.data.getFloat64(this.offset, this.littleEndian); |
|||
this.offset += 8; |
|||
return rv; |
|||
} |
|||
|
|||
private alignInt() { |
|||
if ((this.offset & (this.intAlign - 1)) != 0) { |
|||
this.offset = Math.ceil(this.offset / this.intAlign) * this.intAlign; |
|||
} |
|||
} |
|||
|
|||
private alignFloat() { |
|||
// align?
|
|||
if ((this.offset & (this.floatAlign - 1)) != 0) { |
|||
this.offset = Math.ceil(this.offset / this.floatAlign) * this.floatAlign; |
|||
} |
|||
} |
|||
} |
|||
Binary file not shown.
@ -1,18 +0,0 @@ |
|||
import { arrayGrouped, range } from "../arrays"; |
|||
|
|||
import "should"; |
|||
import "source-map-support/register"; |
|||
|
|||
describe("arrays", () => { |
|||
it("range", () => { |
|||
range(3, 6).should.eql([ 3, 4, 5 ]); |
|||
range(4, 20, 3).should.eql([4, 7, 10, 13, 16, 19]); |
|||
}); |
|||
|
|||
it("arrayGrouped", () => { |
|||
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 7).should.eql([ [ 1, 2, 3, 4, 5, 6, 7 ] ]); |
|||
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 3).should.eql([ [ 1, 2, 3 ], [ 4, 5, 6 ], [ 7 ] ]); |
|||
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 2).should.eql([ [ 1, 2 ], [ 3, 4 ], [ 5, 6 ], [ 7 ] ]); |
|||
arrayGrouped([ 1, 2, 3, 4, 5, 6, 7 ], 1).should.eql([ [ 1 ], [ 2 ], [ 3 ], [ 4 ], [ 5 ], [ 6 ], [ 7 ] ]); |
|||
}); |
|||
}); |
|||
@ -1,80 +0,0 @@ |
|||
import * as fs from "fs"; |
|||
import { RrdFile } from "../rrd"; |
|||
|
|||
import "should"; |
|||
import "source-map-support/register"; |
|||
|
|||
/* |
|||
* ping-later.rrd: 1504399042 |
|||
* rrdtool xport --json -s 1504396100 -e 1504396300 DEF:ping=ping-later.rrd:value:MAX XPORT:ping |
|||
*/ |
|||
|
|||
describe("RrdFile", () => { |
|||
function readFile(filename: string): ArrayBuffer { |
|||
const data = fs.readFileSync(`./src/test/data/${filename}`); |
|||
return data.buffer; |
|||
} |
|||
|
|||
it("validate", () => { |
|||
const rrd = new RrdFile(new DataView(readFile("ping-later.rrd"))); |
|||
rrd.toString().should.eql( |
|||
"RrdFile(LE, version=3, sources=1, archives=15, lastUpdate=1504399040, step=10, int=64:8, float=64:8)" |
|||
); |
|||
rrd.getKeys().should.eql([ "value:AVERAGE", "value:MIN", "value:MAX" ]); |
|||
}); |
|||
|
|||
it("read the finest granularity", () => { |
|||
const rrd = new RrdFile(new DataView(readFile("ping-later.rrd"))); |
|||
rrd.getSeries("value:MAX", 1504399000, 1504399050).should.eql([ |
|||
[ 1504399000, 17.9191 ], |
|||
[ 1504399010, 23.8354 ], |
|||
[ 1504399020, 18.8518 ], |
|||
[ 1504399030, 18.8518 ], |
|||
[ 1504399040, 18.7366 ] |
|||
]); |
|||
}); |
|||
|
|||
it("read nulls", () => { |
|||
const rrd = new RrdFile(new DataView(readFile("ping-later.rrd"))); |
|||
rrd.getSeries("value:MAX", 1504396100, 1504396300).should.eql([ |
|||
[ 1504396100, 25.37455 ], |
|||
[ 1504396110, 25.37455 ], |
|||
[ 1504396120, 24.2829 ], |
|||
[ 1504396130, 20.94845 ], |
|||
[ 1504396140, 20.94845 ], |
|||
[ 1504396150, 25.2886 ], |
|||
[ 1504396160, 25.39535 ], |
|||
[ 1504396170, 25.39535 ], |
|||
[ 1504396180, 24.5029 ], |
|||
[ 1504396190, 16.951 ], |
|||
[ 1504396200, 16.951 ], |
|||
[ 1504396210, 17.748800000000003 ], |
|||
[ 1504396220, 25.719050000000003 ], |
|||
[ 1504396230, 25.719050000000003 ], |
|||
[ 1504396240, 24.9086 ], |
|||
[ 1504396250, null ], |
|||
[ 1504396260, null ], |
|||
[ 1504396270, null ], |
|||
[ 1504396280, null ], |
|||
[ 1504396290, null ] |
|||
]); |
|||
}); |
|||
|
|||
it("use a coarser database if necessary", () => { |
|||
const rrd = new RrdFile(new DataView(readFile("ping-later.rrd"))); |
|||
rrd.getSeries("value:MAX", 1504377000, 1504399000).slice(0, 5).should.eql([ |
|||
[ 1504377060, 32.15455 ], |
|||
[ 1504377130, 23.2284 ], |
|||
[ 1504377200, 26.615200000000005 ], |
|||
[ 1504377270, 23.7565 ], |
|||
[ 1504377340, 30.740400000000005 ] |
|||
]); |
|||
}); |
|||
|
|||
it("use the coarsest database if the user is being unreasonable", () => { |
|||
const rrd = new RrdFile(new DataView(readFile("ping-later.rrd"))); |
|||
rrd.getSeries("value:MAX", 1472778000, 1472810000).slice(0, 5).should.eql([ |
|||
[ 1472779040, null ] |
|||
]); |
|||
}); |
|||
}); |
|||
@ -1,41 +0,0 @@ |
|||
import Dygraph from 'dygraphs'; |
|||
|
|||
export const SECOND = 1000; |
|||
export const MINUTE = 60 * SECOND; |
|||
export const HOUR = 60 * MINUTE; |
|||
export const DAY = 24 * HOUR; |
|||
export const WEEK = 7 * DAY; |
|||
|
|||
export function formatDate(dateMaybe: Date | number, granularity: number) { |
|||
const date = (dateMaybe instanceof Date) ? dateMaybe : new Date(dateMaybe); |
|||
const s = date.toString(); |
|||
const seconds = s.slice(16, 24); // HH:MM:SS
|
|||
const hours = s.slice(16, 21); // HH:MM
|
|||
const weekday = s.slice(0, 3); // Mon
|
|||
const day = s.slice(8, 10); // 05
|
|||
const month = s.slice(4, 7); // Jan
|
|||
const year = s.slice(11, 15); // 2015
|
|||
|
|||
switch (granularity) { |
|||
case Dygraph.SECONDLY: |
|||
case Dygraph.TWO_SECONDLY: |
|||
case Dygraph.FIVE_SECONDLY: |
|||
case Dygraph.TEN_SECONDLY: |
|||
case Dygraph.THIRTY_SECONDLY: |
|||
return seconds; |
|||
case Dygraph.MINUTELY: |
|||
case Dygraph.TWO_MINUTELY: |
|||
case Dygraph.FIVE_MINUTELY: |
|||
case Dygraph.TEN_MINUTELY: |
|||
case Dygraph.THIRTY_MINUTELY: |
|||
return hours; |
|||
case Dygraph.HOURLY: |
|||
case Dygraph.TWO_HOURLY: |
|||
case Dygraph.SIX_HOURLY: |
|||
return hours.slice(0, 2) == "00" ? weekday + "<br>" + day + " " + month : hours; |
|||
case Dygraph.DAILY: |
|||
return weekday + "<br>" + day + " " + month; |
|||
default: // WEEKLY, MONTHLY, ...
|
|||
return day + " " + month + "<br>" + year; |
|||
} |
|||
} |
|||
Loading…
Reference in new issue