From e905618113309eaba7227ff1328a20f6846e4afd Mon Sep 17 00:00:00 2001 From: Emiliano Ciavatta Date: Mon, 5 Oct 2020 23:46:18 +0200 Subject: Implement timeline --- frontend/package.json | 3 + .../src/components/filters/FiltersDefinitions.js | 86 +++--- frontend/src/components/panels/MainPane.js | 34 +-- frontend/src/globals.js | 5 + frontend/src/log.js | 7 + frontend/src/views/App.js | 18 +- frontend/src/views/Connections.js | 158 +++++++---- frontend/src/views/Connections.scss | 52 ++-- frontend/src/views/Footer.js | 173 +++++++++++- frontend/src/views/Footer.scss | 17 +- frontend/yarn.lock | 312 +++++++++++++++++++-- 11 files changed, 670 insertions(+), 195 deletions(-) create mode 100644 frontend/src/globals.js create mode 100644 frontend/src/log.js (limited to 'frontend') diff --git a/frontend/package.json b/frontend/package.json index f22fad5..5bc13f1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,8 +14,10 @@ "classnames": "^2.2.6", "dompurify": "^2.1.1", "eslint-config-react-app": "^5.2.1", + "flux": "^3.1.3", "lodash": "^4.17.20", "node-sass": "^4.14.0", + "pondjs": "^0.9.0", "react": "^16.13.1", "react-bootstrap": "^1.0.1", "react-dom": "^16.13.1", @@ -25,6 +27,7 @@ "react-router-dom": "^5.1.2", "react-scripts": "3.4.1", "react-tag-autocomplete": "^6.0.0-beta.6", + "react-timeseries-charts": "^0.16.1", "typed.js": "^2.0.11" }, "scripts": { diff --git a/frontend/src/components/filters/FiltersDefinitions.js b/frontend/src/components/filters/FiltersDefinitions.js index 02ccb42..d4f2912 100644 --- a/frontend/src/components/filters/FiltersDefinitions.js +++ b/frontend/src/components/filters/FiltersDefinitions.js @@ -1,12 +1,4 @@ -import { - cleanNumber, - timestampToTime, - timeToTimestamp, - validate24HourTime, - validateIpAddress, - validateMin, - validatePort -} from "../../utils"; +import {cleanNumber, validateIpAddress, validateMin, validatePort} from "../../utils"; import StringConnectionsFilter from "./StringConnectionsFilter"; import React from "react"; import RulesConnectionsFilter from "./RulesConnectionsFilter"; @@ -22,69 +14,69 @@ export const filtersDefinitions = { replaceFunc={cleanNumber} validateFunc={validatePort} key="service_port_filter" - width={200} />, - matched_rules: , + width={200}/>, + matched_rules: , client_address: , + width={320}/>, client_port: , + width={200}/>, min_duration: , + width={200}/>, max_duration: , + width={200}/>, min_bytes: , + width={200}/>, max_bytes: , - started_after: , - started_before: , - closed_after: , - closed_before: , - marked: , - hidden: + width={200}/>, + // started_after: , + // started_before: , + // closed_after: , + // closed_before: , + marked: , + // hidden: }; diff --git a/frontend/src/components/panels/MainPane.js b/frontend/src/components/panels/MainPane.js index 3202d6d..bd25e0c 100644 --- a/frontend/src/components/panels/MainPane.js +++ b/frontend/src/components/panels/MainPane.js @@ -8,24 +8,23 @@ import PcapPane from "./PcapPane"; import backend from "../../backend"; import RulePane from "./RulePane"; import ServicePane from "./ServicePane"; +import log from "../../log"; class MainPane extends Component { - constructor(props) { - super(props); - this.state = { - selectedConnection: null, - loading: false - }; - } + state = {}; componentDidMount() { const match = this.props.location.pathname.match(/^\/connections\/([a-f0-9]{24})$/); if (match != null) { - this.setState({loading: true}); + this.loading = true; backend.get(`/api/connections/${match[1]}`) - .then(res => this.setState({selectedConnection: res.json, loading: false})) - .catch(error => console.log(error)); + .then(res => { + this.loading = false; + this.setState({selectedConnection: res.json}); + log.debug(`Initial connection ${match[1]} loaded`); + }) + .catch(error => log.error("Error loading initial connection", error)); } } @@ -34,18 +33,19 @@ class MainPane extends Component {
{ - !this.state.loading && + !this.loading && this.setState({selectedConnection: c})} - initialConnection={this.state.selectedConnection} /> + initialConnection={this.state.selectedConnection}/> }
- } /> - } /> - } /> - } /> - } /> + }/> + }/> + }/> + }/> + }/>
diff --git a/frontend/src/globals.js b/frontend/src/globals.js new file mode 100644 index 0000000..cd4dc64 --- /dev/null +++ b/frontend/src/globals.js @@ -0,0 +1,5 @@ +import {Dispatcher} from "flux"; + +const dispatcher = new Dispatcher(); + +export default dispatcher; diff --git a/frontend/src/log.js b/frontend/src/log.js new file mode 100644 index 0000000..0883962 --- /dev/null +++ b/frontend/src/log.js @@ -0,0 +1,7 @@ +const log = { + debug: (...obj) => console.info(...obj), + info: (...obj) => console.info(...obj), + error: (...obj) => console.error(obj) +}; + +export default log; diff --git a/frontend/src/views/App.js b/frontend/src/views/App.js index abd5209..00d9110 100644 --- a/frontend/src/views/App.js +++ b/frontend/src/views/App.js @@ -7,17 +7,17 @@ import {BrowserRouter as Router} from "react-router-dom"; import Filters from "./Filters"; import backend from "../backend"; import ConfigurationPane from "../components/panels/ConfigurationPane"; +import log from "../log"; class App extends Component { - constructor(props) { - super(props); - - this.state = {}; - } + state = {}; componentDidMount() { - backend.get("/api/services").then(_ => this.setState({configured: true})); + backend.get("/api/services").then(_ => { + log.debug("Caronte is already configured. Loading main.."); + this.setState({configured: true}); + }); setInterval(() => { if (document.title.endsWith("❚")) { @@ -38,11 +38,11 @@ class App extends Component {
-
this.setState({filterWindowOpen: true})} /> +
this.setState({filterWindowOpen: true})}/>
- {this.state.configured ? : - this.setState({configured: true})} />} + {this.state.configured ? : + this.setState({configured: true})}/>} {modal}
diff --git a/frontend/src/views/Connections.js b/frontend/src/views/Connections.js index 73979c4..fe655b3 100644 --- a/frontend/src/views/Connections.js +++ b/frontend/src/views/Connections.js @@ -6,26 +6,31 @@ import {Redirect} from 'react-router'; import {withRouter} from "react-router-dom"; import backend from "../backend"; import ConnectionMatchedRules from "../components/ConnectionMatchedRules"; +import dispatcher from "../globals"; +import log from "../log"; +import ButtonField from "../components/fields/ButtonField"; class Connections extends Component { + state = { + loading: false, + connections: [], + firstConnection: null, + lastConnection: null, + flagRule: null, + rules: null, + queryString: null + }; + constructor(props) { super(props); - this.state = { - loading: false, - connections: [], - firstConnection: null, - lastConnection: null, - prevParams: null, - flagRule: null, - rules: null, - queryString: null - }; this.scrollTopThreashold = 0.00001; this.scrollBottomThreashold = 0.99999; - this.maxConnections = 500; + this.maxConnections = 200; this.queryLimit = 50; + this.connectionsListRef = React.createRef(); + this.lastScrollPosition = 0; } componentDidMount() { @@ -35,6 +40,17 @@ class Connections extends Component { this.setState({selected: this.props.initialConnection.id}); // TODO: scroll to initial connection } + + dispatcher.register((payload) => { + if (payload.actionType === "timeline-update") { + this.connectionsListRef.current.scrollTop = 0; + this.loadConnections({ + started_after: Math.round(payload.from.getTime() / 1000), + started_before: Math.round(payload.to.getTime() / 1000), + limit: this.maxConnections + }).then(() => log.info(`Loading connections between ${payload.from} and ${payload.to}`)); + } + }); } connectionSelected = (c) => { @@ -46,7 +62,7 @@ class Connections extends Component { if (this.state.loaded && prevProps.location.search !== this.props.location.search) { this.setState({queryString: this.props.location.search}); this.loadConnections({limit: this.queryLimit}) - .then(() => console.log("Connections reloaded after query string update")); + .then(() => log.info("Connections reloaded after query string update")); } } @@ -54,12 +70,26 @@ class Connections extends Component { let relativeScroll = e.currentTarget.scrollTop / (e.currentTarget.scrollHeight - e.currentTarget.clientHeight); if (!this.state.loading && relativeScroll > this.scrollBottomThreashold) { this.loadConnections({from: this.state.lastConnection.id, limit: this.queryLimit,}) - .then(() => console.log("Following connections loaded")); + .then(() => log.info("Following connections loaded")); } if (!this.state.loading && relativeScroll < this.scrollTopThreashold) { this.loadConnections({to: this.state.firstConnection.id, limit: this.queryLimit,}) - .then(() => console.log("Previous connections loaded")); + .then(() => log.info("Previous connections loaded")); + if (this.state.showMoreRecentButton) { + this.setState({showMoreRecentButton: false}); + } + } else { + if (this.lastScrollPosition > e.currentTarget.scrollTop) { + if (!this.state.showMoreRecentButton) { + this.setState({showMoreRecentButton: true}); + } + } else { + if (this.state.showMoreRecentButton) { + this.setState({showMoreRecentButton: false}); + } + } } + this.lastScrollPosition = e.currentTarget.scrollTop; }; addServicePortFilter = (port) => { @@ -85,14 +115,14 @@ class Connections extends Component { urlParams.set(name, value); } - this.setState({loading: true, prevParams: params}); + this.setState({loading: true}); let res = (await backend.get(`${url}?${urlParams}`)).json; let connections = this.state.connections; let firstConnection = this.state.firstConnection; let lastConnection = this.state.lastConnection; - if (params !== undefined && params.from !== undefined) { + if (params !== undefined && params.from !== undefined && params.to === undefined) { if (res.length > 0) { connections = this.state.connections.concat(res); lastConnection = connections[connections.length - 1]; @@ -102,7 +132,7 @@ class Connections extends Component { firstConnection = connections[0]; } } - } else if (params !== undefined && params.to !== undefined) { + } else if (params !== undefined && params.to !== undefined && params.from === undefined) { if (res.length > 0) { connections = res.concat(this.state.connections); firstConnection = connections[0]; @@ -112,6 +142,7 @@ class Connections extends Component { } } } else { + this.connectionsListRef.current.scrollTop = 0; if (res.length > 0) { connections = res; firstConnection = connections[0]; @@ -124,7 +155,7 @@ class Connections extends Component { } let rules = this.state.rules; - if (rules === null) { + if (rules == null) { rules = (await backend.get("/api/rules")).json; } @@ -135,15 +166,21 @@ class Connections extends Component { firstConnection: firstConnection, lastConnection: lastConnection }); + + if (firstConnection != null && lastConnection != null) { + dispatcher.dispatch({ + actionType: "connections-update", + from: new Date(lastConnection["started_at"]), + to: new Date(firstConnection["started_at"]) + }); + } } render() { let redirect; let queryString = this.state.queryString !== null ? this.state.queryString : ""; if (this.state.selected) { - let format = this.props.match.params.format; - format = format !== undefined ? "/" + format : ""; - redirect = ; + redirect = ; } let loading = null; @@ -154,48 +191,55 @@ class Connections extends Component { } return ( -
-
- - - - - - - - - - - - - - - - - { - this.state.connections.flatMap(c => { - return [ this.connectionSelected(c)} - selected={this.state.selected === c.id} - onMarked={marked => c.marked = marked} - onEnabled={enabled => c.hidden = !enabled} - addServicePortFilter={this.addServicePortFilter} />, - c.matched_rules.length > 0 && +
+ {this.state.showMoreRecentButton &&
+ + this.loadConnections({limit: this.queryLimit}) + .then(() => log.info("Most recent connections loaded")) + }/> +
} + +
+
servicesrcipsrcportdstipdstportstarted_atdurationupdownactions
+ + + + + + + + + + + + + + + + { + this.state.connections.flatMap(c => { + return [ this.connectionSelected(c)} + selected={this.state.selected === c.id} + onMarked={marked => c.marked = marked} + onEnabled={enabled => c.hidden = !enabled} + addServicePortFilter={this.addServicePortFilter}/>, + c.matched_rules.length > 0 && - ]; - }) - } - {loading} - -
servicesrcipsrcportdstipdstportstarted_atdurationupdownactions
- - {redirect} + addMatchedRulesFilter={this.addMatchedRulesFilter}/> + ]; + }) + } + {loading} + + + + {redirect} +
); } } - export default withRouter(Connections); diff --git a/frontend/src/views/Connections.scss b/frontend/src/views/Connections.scss index c7bd1df..9e2b6ba 100644 --- a/frontend/src/views/Connections.scss +++ b/frontend/src/views/Connections.scss @@ -1,37 +1,39 @@ @import "../colors.scss"; -.connections { +.connections-container { position: relative; - overflow: auto; height: 100%; - padding: 0 10px; + padding: 10px; background-color: $color-primary-3; - .table { - margin-top: 10px; - } + .connections { + position: relative; + overflow-y: scroll; + height: 100%; - .connections-header-padding { - position: sticky; - top: 0; - right: 0; - left: 0; - height: 10px; - margin-bottom: -10px; - background-color: $color-primary-3; - } + .table { + margin-bottom: 0; + } + + th { + font-size: 13.5px; + position: sticky; + top: 0; + padding: 5px; + border: none; + background-color: $color-primary-2; + } - th { - font-size: 13.5px; - position: sticky; - top: 10px; - padding: 5px; - border-top: 3px solid $color-primary-3; - border-bottom: 3px solid $color-primary-3; - background-color: $color-primary-2; + &:hover::-webkit-scrollbar-thumb { + background: $color-secondary-2; + } } - &:hover::-webkit-scrollbar-thumb { - background: $color-secondary-2; + .most-recent-button { + position: absolute; + z-index: 20; + top: 45px; + left: calc(50% - 50px); + background-color: red; } } diff --git a/frontend/src/views/Footer.js b/frontend/src/views/Footer.js index 0a3c5a3..ad1e4f7 100644 --- a/frontend/src/views/Footer.js +++ b/frontend/src/views/Footer.js @@ -1,19 +1,180 @@ import React, {Component} from 'react'; import './Footer.scss'; +import { + ChartContainer, + ChartRow, + Charts, + LineChart, + MultiBrush, + Resizable, + styler, + YAxis +} from "react-timeseries-charts"; +import {TimeRange, TimeSeries} from "pondjs"; +import backend from "../backend"; +import ChoiceField from "../components/fields/ChoiceField"; +import dispatcher from "../globals"; +import {withRouter} from "react-router-dom"; +import log from "../log"; + class Footer extends Component { + state = { + metric: "connections_per_service" + }; + + constructor() { + super(); + + this.disableTimeSeriesChanges = false; + this.selectionTimeout = null; + } + + filteredPort = () => { + const urlParams = new URLSearchParams(this.props.location.search); + return urlParams.get("service_port"); + }; + + componentDidMount() { + const filteredPort = this.filteredPort(); + this.setState({filteredPort}); + this.loadStatistics(this.state.metric, filteredPort).then(() => + log.debug("Statistics loaded after mount")); + + dispatcher.register((payload) => { + if (payload.actionType === "connections-update") { + this.setState({ + selection: new TimeRange(payload.from, payload.to), + }); + } + }); + } + + componentDidUpdate(prevProps, prevState, snapshot) { + const filteredPort = this.filteredPort(); + if (this.state.filteredPort !== filteredPort) { + this.setState({filteredPort}); + this.loadStatistics(this.state.metric, filteredPort).then(() => + log.debug("Statistics reloaded after filtered port changes")); + } + } + + loadStatistics = async (metric, filteredPort) => { + const urlParams = new URLSearchParams(); + urlParams.set("metric", metric); + + let services = (await backend.get("/api/services")).json; + if (filteredPort && services[filteredPort]) { + const service = services[filteredPort]; + services = {}; + services[filteredPort] = service; + } + + const ports = Object.keys(services); + ports.forEach(s => urlParams.append("ports", s)); + + const metrics = (await backend.get("/api/statistics?" + urlParams)).json; + const series = new TimeSeries({ + name: "statistics", + columns: ["time"].concat(ports), + points: metrics.map(m => [new Date(m["range_start"])].concat(ports.map(p => m[metric][p] || 0))) + }); + this.setState({ + metric, + series, + services, + timeRange: series.range(), + }); + log.debug(`Loaded statistics for metric "${metric}" for services [${ports}]`); + }; + + createStyler = () => { + return styler(Object.keys(this.state.services).map(port => { + return {key: port, color: this.state.services[port].color, width: 2}; + })); + }; + + handleTimeRangeChange = (timeRange) => { + if (!this.disableTimeSeriesChanges) { + this.setState({timeRange}); + } + }; + + handleSelectionChange = (timeRange) => { + this.disableTimeSeriesChanges = true; + + this.setState({selection: timeRange}); + if (this.selectionTimeout) { + clearTimeout(this.selectionTimeout); + } + this.selectionTimeout = setTimeout(() => { + dispatcher.dispatch({ + actionType: "timeline-update", + from: timeRange.begin(), + to: timeRange.end() + }); + this.selectionTimeout = null; + this.disableTimeSeriesChanges = false; + }, 1000); + }; + + aggregateSeries = (func) => { + const values = this.state.series.columns().map(c => this.state.series[func](c)); + return Math[func](...values); + }; + render() { return ( -