diff options
| -rw-r--r-- | connection_streams_controller.go | 78 | ||||
| -rw-r--r-- | frontend/src/components/Connection.js | 7 | ||||
| -rw-r--r-- | frontend/src/components/ConnectionContent.js | 163 | ||||
| -rw-r--r-- | frontend/src/components/ConnectionContent.scss | 105 | ||||
| -rw-r--r-- | frontend/src/components/MessageAction.js | 52 | ||||
| -rw-r--r-- | frontend/src/components/MessageAction.scss | 8 | ||||
| -rw-r--r-- | frontend/src/utils.js | 5 | ||||
| -rw-r--r-- | go.mod | 1 | ||||
| -rw-r--r-- | go.sum | 2 | ||||
| -rw-r--r-- | parsers/http_request_parser.go | 149 | ||||
| -rw-r--r-- | parsers/http_response_parser.go | 77 | ||||
| -rw-r--r-- | parsers/parser.go | 28 | ||||
| -rw-r--r-- | parsers/parser_utils.go | 24 |
13 files changed, 639 insertions, 60 deletions
diff --git a/connection_streams_controller.go b/connection_streams_controller.go index 251e842..9d73b0e 100644 --- a/connection_streams_controller.go +++ b/connection_streams_controller.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "context" + "github.com/eciavatta/caronte/parsers" log "github.com/sirupsen/logrus" "time" ) @@ -25,12 +27,14 @@ type ConnectionStream struct { type PatternSlice [2]uint64 type Payload struct { - FromClient bool `json:"from_client"` - Content string `json:"content"` - Index int `json:"index"` - Timestamp time.Time `json:"timestamp"` - IsRetransmitted bool `json:"is_retransmitted"` - RegexMatches []RegexSlice `json:"regex_matches"` + FromClient bool `json:"from_client"` + Content string `json:"content"` + Metadata parsers.Metadata `json:"metadata"` + IsMetadataContinuation bool `json:"is_metadata_continuation"` + Index int `json:"index"` + Timestamp time.Time `json:"timestamp"` + IsRetransmitted bool `json:"is_retransmitted"` + RegexMatches []RegexSlice `json:"regex_matches"` } type RegexSlice struct { @@ -55,8 +59,8 @@ func NewConnectionStreamsController(storage Storage) ConnectionStreamsController } func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, connectionID RowID, - format QueryFormat) []Payload { - payloads := make([]Payload, 0, InitialPayloadsSize) + format QueryFormat) []*Payload { + payloads := make([]*Payload, 0, InitialPayloadsSize) var clientIndex, serverIndex, globalIndex uint64 if format.Limit <= 0 { @@ -75,7 +79,11 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c return serverBlocksIndex < len(serverStream.BlocksIndexes) } - var payload Payload + var payload *Payload + payloadsBuffer := make([]*Payload, 0, 16) + contentChunkBuffer := new(bytes.Buffer) + var lastContentSlice []byte + var sideChanged, lastClient, lastServer bool for !clientStream.ID.IsZero() || !serverStream.ID.IsZero() { if hasClientBlocks() && (!hasServerBlocks() || // next payload is from client clientStream.BlocksTimestamps[clientBlocksIndex].UnixNano() <= @@ -85,11 +93,11 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c if clientBlocksIndex < len(clientStream.BlocksIndexes)-1 { end = clientStream.BlocksIndexes[clientBlocksIndex+1] } else { - end = len(clientStream.Payload) - 1 + end = len(clientStream.Payload) } size := uint64(end - start) - payload = Payload{ + payload = &Payload{ FromClient: true, Content: DecodeBytes(clientStream.Payload[start:end], format.Format), Index: start, @@ -100,17 +108,20 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c clientIndex += size globalIndex += size clientBlocksIndex++ + + lastContentSlice = clientStream.Payload[start:end] + sideChanged, lastClient, lastServer = lastServer, true, false } else { // next payload is from server start := serverStream.BlocksIndexes[serverBlocksIndex] end := 0 if serverBlocksIndex < len(serverStream.BlocksIndexes)-1 { end = serverStream.BlocksIndexes[serverBlocksIndex+1] } else { - end = len(serverStream.Payload) - 1 + end = len(serverStream.Payload) } size := uint64(end - start) - payload = Payload{ + payload = &Payload{ FromClient: false, Content: DecodeBytes(serverStream.Payload[start:end], format.Format), Index: start, @@ -121,13 +132,9 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c serverIndex += size globalIndex += size serverBlocksIndex++ - } - if globalIndex > format.Skip { - payloads = append(payloads, payload) - } - if globalIndex > format.Skip+format.Limit { - return payloads + lastContentSlice = serverStream.Payload[start:end] + sideChanged, lastClient, lastServer = lastClient, false, true } if !hasClientBlocks() { @@ -140,6 +147,39 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c serverBlocksIndex = 0 serverStream = csc.getConnectionStream(c, connectionID, false, serverDocumentIndex) } + + updateMetadata := func() { + metadata := parsers.Parse(contentChunkBuffer.Bytes()) + var isMetadataContinuation bool + for _, elem := range payloadsBuffer { + elem.Metadata = metadata + elem.IsMetadataContinuation = isMetadataContinuation + isMetadataContinuation = true + } + + payloadsBuffer = payloadsBuffer[:0] + contentChunkBuffer.Reset() + } + + if sideChanged { + updateMetadata() + } + payloadsBuffer = append(payloadsBuffer, payload) + contentChunkBuffer.Write(lastContentSlice) + + if clientStream.ID.IsZero() && serverStream.ID.IsZero() { + updateMetadata() + } + + if globalIndex > format.Skip { + // problem: waste of time if the payload is discarded + payloads = append(payloads, payload) + } + if globalIndex > format.Skip+format.Limit { + // problem: the last chunk is not parsed, but can be ok because it is not finished + updateMetadata() + return payloads + } } return payloads diff --git a/frontend/src/components/Connection.js b/frontend/src/components/Connection.js index e41f542..93c6438 100644 --- a/frontend/src/components/Connection.js +++ b/frontend/src/components/Connection.js @@ -57,6 +57,11 @@ class Connection extends Component { let closedAt = new Date(conn.closed_at); let processedAt = new Date(conn.processed_at); let duration = ((closedAt - startedAt) / 1000).toFixed(3); + if (duration > 1000 || duration < -1000) { + duration = "∞"; + } else { + duration += "s"; + } let timeInfo = <div> <span>Started at {startedAt.toLocaleDateString() + " " + startedAt.toLocaleTimeString()}</span><br/> <span>Processed at {processedAt.toLocaleDateString() + " " + processedAt.toLocaleTimeString()}</span><br/> @@ -106,7 +111,7 @@ class Connection extends Component { <td className="clickable" onClick={this.props.onSelected}> <OverlayTrigger trigger={["focus", "hover"]} placement="right" overlay={popoverFor("duration", timeInfo)}> - <span className="test-tooltip">{duration}s</span> + <span className="test-tooltip">{duration}</span> </OverlayTrigger> </td> <td className="clickable" onClick={this.props.onSelected}>{conn.client_bytes}</td> diff --git a/frontend/src/components/ConnectionContent.js b/frontend/src/components/ConnectionContent.js index 905a56d..20ec92b 100644 --- a/frontend/src/components/ConnectionContent.js +++ b/frontend/src/components/ConnectionContent.js @@ -1,7 +1,10 @@ import React, {Component} from 'react'; import './ConnectionContent.scss'; -import {Dropdown} from 'react-bootstrap'; +import {Button, Dropdown, Row} from 'react-bootstrap'; import axios from 'axios'; +import MessageAction from "./MessageAction"; + +const classNames = require('classnames'); class ConnectionContent extends Component { @@ -10,7 +13,9 @@ class ConnectionContent extends Component { this.state = { loading: false, connectionContent: null, - format: "default" + format: "default", + tryParse: true, + messageActionDialog: null }; this.validFormats = ["default", "hex", "hexdump", "base32", "base64", "ascii", "binary", "decimal", "octal"]; @@ -37,44 +42,148 @@ class ConnectionContent extends Component { } } + tryParseConnectionMessage(connectionMessage) { + if (connectionMessage.metadata == null) { + return connectionMessage.content; + } + if (connectionMessage["is_metadata_continuation"]) { + return <span style={{"fontSize": "12px"}}>**already parsed in previous messages**</span>; + } + + let unrollMap = (obj) => obj == null ? null : Object.entries(obj).map(([key, value]) => + <p><strong>{key}</strong>: {value}</p> + ); + + let m = connectionMessage.metadata; + switch (m.type) { + case "http-request": + let url = <i><u><a href={"http://" + m.host + m.url} target="_blank" + rel="noopener noreferrer">{m.host}{m.url}</a></u></i>; + return <span className="type-http-request"> + <p style={{"marginBottom": "7px"}}><strong>{m.method}</strong> {url} {m.protocol}</p> + {unrollMap(m.headers)} + <div style={{"margin": "20px 0"}}>{m.body}</div> + {unrollMap(m.trailers)} + </span>; + case "http-response": + return <span className="type-http-response"> + <p style={{"marginBottom": "7px"}}>{m.protocol} <strong>{m.status}</strong></p> + {unrollMap(m.headers)} + <div style={{"margin": "20px 0"}}>{m.body}</div> + {unrollMap(m.trailers)} + </span>; + default: + return connectionMessage.content; + } + } + + connectionsActions(connectionMessage) { + if (connectionMessage.metadata == null || connectionMessage.metadata["reproducers"] === undefined) { + return null; + } + + return Object.entries(connectionMessage.metadata["reproducers"]).map(([actionName, actionValue]) => + <Button size="sm" key={actionName + "_button"} onClick={() => { + this.setState({ + messageActionDialog: <MessageAction actionName={actionName} actionValue={actionValue} + onHide={() => this.setState({messageActionDialog: null})}/> + }); + }}>{actionName}</Button> + ); + } + render() { let content = this.state.connectionContent; - if (content === null) { - return <div>nope</div>; + if (content == null) { + return <div>select a connection to view</div>; } let payload = content.map((c, i) => - <span key={`content-${i}`} className={c.from_client ? "from-client" : "from-server"}> - {c.content} - </span> + <div key={`content-${i}`} + className={classNames("connection-message", c.from_client ? "from-client" : "from-server")}> + <div className="connection-message-header container-fluid"> + <div className="row"> + <div className="connection-message-info col"> + <span><strong>offset</strong>: {c.index}</span> | <span><strong>timestamp</strong>: {c.timestamp} + </span> | <span><strong>retransmitted</strong>: {c["is_retransmitted"] ? "yes" : "no"}</span> + </div> + <div className="connection-message-actions col-auto">{this.connectionsActions(c)}</div> + </div> + </div> + <div className="connection-message-label">{c.from_client ? "client" : "server"}</div> + <div + className={classNames("message-content", this.state.decoded ? "message-parsed" : "message-original")}> + {this.state.tryParse && this.state.format === "default" ? this.tryParseConnectionMessage(c) : c.content} + </div> + </div> ); return ( <div className="connection-content"> - <div className="connection-content-options"> - <Dropdown onSelect={this.setFormat} > - <Dropdown.Toggle size="sm" id="dropdown-basic"> - format - </Dropdown.Toggle> - - <Dropdown.Menu> - <Dropdown.Item eventKey="default" active={this.state.format === "default"}>plain</Dropdown.Item> - <Dropdown.Item eventKey="hex" active={this.state.format === "hex"}>hex</Dropdown.Item> - <Dropdown.Item eventKey="hexdump" active={this.state.format === "hexdump"}>hexdump</Dropdown.Item> - <Dropdown.Item eventKey="base32" active={this.state.format === "base32"}>base32</Dropdown.Item> - <Dropdown.Item eventKey="base64" active={this.state.format === "base64"}>base64</Dropdown.Item> - <Dropdown.Item eventKey="ascii" active={this.state.format === "ascii"}>ascii</Dropdown.Item> - <Dropdown.Item eventKey="binary" active={this.state.format === "binary"}>binary</Dropdown.Item> - <Dropdown.Item eventKey="decimal" active={this.state.format === "decimal"}>decimal</Dropdown.Item> - <Dropdown.Item eventKey="octal" active={this.state.format === "octal"}>octal</Dropdown.Item> - </Dropdown.Menu> - </Dropdown> - - + <div className="connection-content-header container-fluid"> + <Row> + <div className="header-info col"> + <span><strong>flow</strong>: {this.props.connection.ip_src}:{this.props.connection.port_src} -> {this.props.connection.ip_dst}:{this.props.connection.port_dst}</span> + <span> | <strong>timestamp</strong>: {this.props.connection.started_at}</span> + </div> + <div className="header-actions col-auto"> + <Dropdown onSelect={this.setFormat}> + <Dropdown.Toggle size="sm" id="connection-content-format"> + format + </Dropdown.Toggle> + + <Dropdown.Menu> + <Dropdown.Item eventKey="default" + active={this.state.format === "default"}>plain</Dropdown.Item> + <Dropdown.Item eventKey="hex" + active={this.state.format === "hex"}>hex</Dropdown.Item> + <Dropdown.Item eventKey="hexdump" + active={this.state.format === "hexdump"}>hexdump</Dropdown.Item> + <Dropdown.Item eventKey="base32" + active={this.state.format === "base32"}>base32</Dropdown.Item> + <Dropdown.Item eventKey="base64" + active={this.state.format === "base64"}>base64</Dropdown.Item> + <Dropdown.Item eventKey="ascii" + active={this.state.format === "ascii"}>ascii</Dropdown.Item> + <Dropdown.Item eventKey="binary" + active={this.state.format === "binary"}>binary</Dropdown.Item> + <Dropdown.Item eventKey="decimal" + active={this.state.format === "decimal"}>decimal</Dropdown.Item> + <Dropdown.Item eventKey="octal" + active={this.state.format === "octal"}>octal</Dropdown.Item> + </Dropdown.Menu> + </Dropdown> + + <Dropdown> + <Dropdown.Toggle size="sm" id="connection-content-view"> + view_as + </Dropdown.Toggle> + + <Dropdown.Menu> + <Dropdown.Item eventKey="default" active={true}>default</Dropdown.Item> + </Dropdown.Menu> + + </Dropdown> + + <Dropdown> + <Dropdown.Toggle size="sm" id="connection-content-download"> + download_as + </Dropdown.Toggle> + + <Dropdown.Menu> + <Dropdown.Item eventKey="nl_separated">nl_separated</Dropdown.Item> + <Dropdown.Item eventKey="only_client">only_client</Dropdown.Item> + <Dropdown.Item eventKey="only_server">only_server</Dropdown.Item> + </Dropdown.Menu> + + </Dropdown> + </div> + </Row> </div> <pre>{payload}</pre> + {this.state.messageActionDialog} </div> ); } diff --git a/frontend/src/components/ConnectionContent.scss b/frontend/src/components/ConnectionContent.scss index a1b4afd..8ee31ec 100644 --- a/frontend/src/components/ConnectionContent.scss +++ b/frontend/src/components/ConnectionContent.scss @@ -1,29 +1,108 @@ @import '../colors.scss'; .connection-content { - background-color: $color-primary-3; + background-color: $color-primary-0; + padding: 10px 10px 0; height: 100%; - overflow: auto; pre { - background-color: $color-primary-0; - padding: 10px 20px; word-break: break-word; - max-width: 100%; white-space: pre-wrap; - height: 100%; - } + overflow-x: hidden; + height: calc(100% - 31px); + padding: 0 10px; - .from-client { - color: #d4e0fc; + p { + margin: 0; + padding: 0; + } } - .from-server { - color: $color-secondary-4; + .connection-message { + border: 4px solid $color-primary-3; + border-top: 0; + margin: 10px 0; + position: relative; + + .connection-message-header { + background-color: $color-primary-3; + height: 25px; + + .connection-message-info { + font-size: 11px; + margin-left: -10px; + margin-top: 6px; + } + + .connection-message-actions { + margin-right: -18px; + display: none; + + button { + margin: 0 3px; + font-size: 11px; + padding: 5px; + } + } + } + + .message-content { + padding: 10px; + } - &:hover { + &:hover .connection-message-actions { + display: block; + } + + .connection-message-label { + position: absolute; background-color: $color-primary-3; - border-top: 1px solid $color-primary-1; + top: 0; + padding: 10px 0; + font-size: 12px; + + writing-mode: vertical-rl; + text-orientation: mixed; + } + + &.from-client { + color: $color-primary-4; + margin-right: 100px; + + .connection-message-label { + right: -22px; + } + } + + &.from-server { + color: $color-primary-4; + margin-left: 100px; + + .connection-message-label { + left: -22px; + transform: rotate(-180deg); + } + } + + } + + .connection-content-header { + background-color: $color-primary-2; + padding: 0; + height: 31px; + + .header-info { + padding-top: 5px; + padding-left: 20px; + font-size: 12px; + } + + .header-actions { + .dropdown { + display: inline-block; + } } } + + } diff --git a/frontend/src/components/MessageAction.js b/frontend/src/components/MessageAction.js new file mode 100644 index 0000000..2c85d84 --- /dev/null +++ b/frontend/src/components/MessageAction.js @@ -0,0 +1,52 @@ +import React, {Component} from 'react'; +import './MessageAction.scss'; +import {Button, FormControl, InputGroup, Modal} from "react-bootstrap"; + +class MessageAction extends Component { + + constructor(props) { + super(props); + this.state = { + copyButtonText: "copy" + }; + this.actionValue = React.createRef(); + this.copyActionValue = this.copyActionValue.bind(this); + } + + copyActionValue() { + this.actionValue.current.select(); + document.execCommand('copy'); + this.setState({copyButtonText: "copied!"}); + setTimeout(() => this.setState({copyButtonText: "copy"}), 3000); + } + + render() { + return ( + <Modal + {...this.props} + show="true" + size="lg" + aria-labelledby="message-action-dialog" + centered + > + <Modal.Header> + <Modal.Title id="message-action-dialog"> + {this.props.actionName} + </Modal.Title> + </Modal.Header> + <Modal.Body> + <InputGroup> + <FormControl as="textarea" className="message-action-value" readOnly={true} + style={{"height": "300px"}} value={this.props.actionValue} ref={this.actionValue} /> + </InputGroup> + </Modal.Body> + <Modal.Footer className="dialog-footer"> + <Button variant="green" onClick={this.copyActionValue}>{this.state.copyButtonText}</Button> + <Button variant="red" onClick={this.props.onHide}>close</Button> + </Modal.Footer> + </Modal> + ); + } +} + +export default MessageAction; diff --git a/frontend/src/components/MessageAction.scss b/frontend/src/components/MessageAction.scss new file mode 100644 index 0000000..f3a8772 --- /dev/null +++ b/frontend/src/components/MessageAction.scss @@ -0,0 +1,8 @@ +@import '../colors.scss'; + +.message-action-value { + font-size: 13px; + padding: 15px; + background-color: $color-primary-2; + color: $color-primary-4; +}
\ No newline at end of file diff --git a/frontend/src/utils.js b/frontend/src/utils.js index 26c10d3..7381f69 100644 --- a/frontend/src/utils.js +++ b/frontend/src/utils.js @@ -59,3 +59,8 @@ export function timestampToTime(timestamp) { let seconds = "0" + d.getSeconds(); return hours + ':' + minutes.substr(-2) + ':' + seconds.substr(-2); } + +export function timestampToDateTime(timestamp) { + let d = new Date(timestamp); + return d.toLocaleDateString() + " " + d.toLocaleTimeString(); +} @@ -17,4 +17,5 @@ require ( go.mongodb.org/mongo-driver v1.3.1 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect + moul.io/http2curl v1.0.0 ) @@ -179,3 +179,5 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= diff --git a/parsers/http_request_parser.go b/parsers/http_request_parser.go new file mode 100644 index 0000000..e2224b8 --- /dev/null +++ b/parsers/http_request_parser.go @@ -0,0 +1,149 @@ +package parsers + +import ( + "bufio" + "bytes" + "encoding/json" + log "github.com/sirupsen/logrus" + "io/ioutil" + "moul.io/http2curl" + "net/http" + "strings" +) + +type HttpRequestMetadata struct { + BasicMetadata + Method string `json:"method"` + URL string `json:"url"` + Protocol string `json:"protocol"` + Host string `json:"host"` + Headers map[string]string `json:"headers"` + Cookies map[string]string `json:"cookies" binding:"omitempty"` + ContentLength int64 `json:"content_length"` + FormData map[string]string `json:"form_data" binding:"omitempty"` + Body string `json:"body" binding:"omitempty"` + Trailer map[string]string `json:"trailer" binding:"omitempty"` + Reproducers HttpRequestMetadataReproducers `json:"reproducers"` +} + +type HttpRequestMetadataReproducers struct { + CurlCommand string `json:"curl_command"` + RequestsCode string `json:"requests_code"` + FetchRequest string `json:"fetch_request"` +} + +type HttpRequestParser struct { +} + +func (p HttpRequestParser) TryParse(content []byte) Metadata { + reader := bufio.NewReader(bytes.NewReader(content)) + request, err := http.ReadRequest(reader) + if err != nil { + return nil + } + var body string + if buffer, err := ioutil.ReadAll(request.Body); err == nil { + body = string(buffer) + } else { + log.WithError(err).Error("failed to read body in http_request_parser") + return nil + } + _ = request.Body.Close() + _ = request.ParseForm() + + return HttpRequestMetadata{ + BasicMetadata: BasicMetadata{"http-request"}, + Method: request.Method, + URL: request.URL.String(), + Protocol: request.Proto, + Host: request.Host, + Headers: JoinArrayMap(request.Header), + Cookies: CookiesMap(request.Cookies()), + ContentLength: request.ContentLength, + FormData: JoinArrayMap(request.Form), + Body: body, + Trailer: JoinArrayMap(request.Trailer), + Reproducers: HttpRequestMetadataReproducers{ + CurlCommand: curlCommand(content), + RequestsCode: requestsCode(request), + FetchRequest: fetchRequest(request, body), + }, + } +} + +func curlCommand(content []byte) string { + // a new reader is required because all the body is read before and GetBody() doesn't works + reader := bufio.NewReader(bytes.NewReader(content)) + request, _ := http.ReadRequest(reader) + if command, err := http2curl.GetCurlCommand(request); err == nil { + return command.String() + } else { + return err.Error() + } +} + +func requestsCode(request *http.Request) string { + var b strings.Builder + var params string + if request.Form != nil { + params = toJson(JoinArrayMap(request.PostForm)) + } + headers := toJson(JoinArrayMap(request.Header)) + cookies := toJson(CookiesMap(request.Cookies())) + + b.WriteString("import requests\n\nresponse = requests." + strings.ToLower(request.Method) + "(") + b.WriteString("\"" + request.URL.String() + "\"") + if params != "" { + b.WriteString(", data = " + params) + } + if headers != "" { + b.WriteString(", headers = " + headers) + } + if cookies != "" { + b.WriteString(", cookies = " + cookies) + } + b.WriteString(")\n") + b.WriteString(` +# print(response.url) +# print(response.text) +# print(response.content) +# print(response.json()) +# print(response.raw) +# print(response.status_code) +# print(response.cookies) +# print(response.history) +`) + + return b.String() +} + +func fetchRequest(request *http.Request, body string) string { + headers := JoinArrayMap(request.Header) + data := make(map[string]interface{}) + data["headers"] = headers + if referrer := request.Header.Get("referrer"); referrer != "" { + data["Referrer"] = referrer + } + // TODO: referrerPolicy + if body == "" { + data["body"] = nil + } else { + data["body"] = body + } + data["method"] = request.Method + // TODO: mode + + if jsonData := toJson(data); jsonData != "" { + return "fetch(\"" + request.URL.String() + "\", " + jsonData + ");" + } else { + return "invalid-request" + } +} + +func toJson(obj interface{}) string { + if buffer, err := json.MarshalIndent(obj, "", "\t"); err == nil { + return string(buffer) + } else { + return "" + } +} diff --git a/parsers/http_response_parser.go b/parsers/http_response_parser.go new file mode 100644 index 0000000..1770116 --- /dev/null +++ b/parsers/http_response_parser.go @@ -0,0 +1,77 @@ +package parsers + +import ( + "bufio" + "bytes" + "compress/gzip" + log "github.com/sirupsen/logrus" + "io/ioutil" + "net/http" +) + +type HttpResponseMetadata struct { + BasicMetadata + Status string `json:"status"` + StatusCode int `json:"status_code"` + Protocol string `json:"protocol"` + Headers map[string]string `json:"headers"` + ConnectionClosed bool `json:"connection_closed"` + Cookies map[string]string `json:"cookies" binding:"omitempty"` + Location string `json:"location" binding:"omitempty"` + Compressed bool `json:"compressed"` + Body string `json:"body" binding:"omitempty"` + Trailer map[string]string `json:"trailer" binding:"omitempty"` +} + +type HttpResponseParser struct { +} + +func (p HttpResponseParser) TryParse(content []byte) Metadata { + reader := bufio.NewReader(bytes.NewReader(content)) + response, err := http.ReadResponse(reader, nil) + if err != nil { + return nil + } + var body string + var compressed bool + switch response.Header.Get("Content-Encoding") { + case "gzip": + if gzipReader, err := gzip.NewReader(response.Body); err == nil { + if buffer, err := ioutil.ReadAll(gzipReader); err == nil { + body = string(buffer) + compressed = true + } else { + log.WithError(err).Error("failed to read gzipped body in http_response_parser") + return nil + } + _ = gzipReader.Close() + } + default: + if buffer, err := ioutil.ReadAll(response.Body); err == nil { + body = string(buffer) + } else { + log.WithError(err).Error("failed to read body in http_response_parser") + return nil + } + } + _ = response.Body.Close() + + var location string + if locationUrl, err := response.Location(); err == nil { + location = locationUrl.String() + } + + return HttpResponseMetadata{ + BasicMetadata: BasicMetadata{"http-response"}, + Status: response.Status, + StatusCode: response.StatusCode, + Protocol: response.Proto, + Headers: JoinArrayMap(response.Header), + ConnectionClosed: response.Close, + Cookies: CookiesMap(response.Cookies()), + Location: location, + Compressed: compressed, + Body: body, + Trailer: JoinArrayMap(response.Trailer), + } +} diff --git a/parsers/parser.go b/parsers/parser.go new file mode 100644 index 0000000..06cc0dc --- /dev/null +++ b/parsers/parser.go @@ -0,0 +1,28 @@ +package parsers + +type Parser interface { + TryParse(content []byte) Metadata + +} + +type Metadata interface { +} + +type BasicMetadata struct { + Type string `json:"type"` +} + +var parsers = []Parser{ // order matter + HttpRequestParser{}, + HttpResponseParser{}, +} + +func Parse(content []byte) Metadata { + for _, parser := range parsers { + if metadata := parser.TryParse(content); metadata != nil { + return metadata + } + } + + return nil +} diff --git a/parsers/parser_utils.go b/parsers/parser_utils.go new file mode 100644 index 0000000..b688262 --- /dev/null +++ b/parsers/parser_utils.go @@ -0,0 +1,24 @@ +package parsers + +import ( + "net/http" + "strings" +) + +func JoinArrayMap(obj map[string][]string) map[string]string { + headers := make(map[string]string, len(obj)) + for key, value := range obj { + headers[key] = strings.Join(value, ";") + } + + return headers +} + +func CookiesMap(cookiesArray []*http.Cookie) map[string]string { + cookies := make(map[string]string, len(cookiesArray)) + for _, cookie := range cookiesArray { + cookies[cookie.Name] = cookie.Value + } + + return cookies +} |