Merge branch 'develop' into feature/frontend

15 files changed, 661 insertions, 175 deletions

diff --git a/Dockerfile b/Dockerfile
index 0092628..e3c49bc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,12 @@
-FROM ubuntu:20.04
+# BUILD STAGE
+FROM ubuntu:20.04 AS BUILDSTAGE
 
 # Install tools and libraries
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -qq git golang-1.14 pkg-config libpcap-dev libhyperscan-dev yarnpkg
+RUN apt-get update && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -qq git golang-1.14 pkg-config libpcap-dev libhyperscan-dev yarnpkg curl
 
 RUN ln -sf ../lib/go-1.14/bin/go /usr/bin/go
 
-ENV GIN_MODE release
 
 COPY . /caronte
 
@@ -13,6 +14,25 @@ WORKDIR /caronte
 
 RUN go mod download && go build
 
-RUN cd frontend && yarnpkg install && yarnpkg build
+RUN cd frontend && \
+	yarnpkg install && \
+	yarnpkg build --production=true
+RUN curl -sf https://gobinaries.com/tj/node-prune | sh && cd /caronte/frontend && node-prune
+
+
+# LAST STAGE
+FROM ubuntu:20.04
+
+COPY --from=BUILDSTAGE /caronte/caronte /caronte/caronte
+COPY --from=BUILDSTAGE /caronte/frontend /caronte/frontend
+COPY --from=BUILDSTAGE /caronte/shared /caronte/shared
+
+RUN apt-get update && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -qq libpcap-dev libhyperscan-dev && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV GIN_MODE release
+
+WORKDIR /caronte
 
 CMD ./caronte
diff --git a/connection_streams_controller.go b/connection_streams_controller.go
index 096210e..9d73b0e 100644
--- a/connection_streams_controller.go
+++ b/connection_streams_controller.go
@@ -1,7 +1,9 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"github.com/eciavatta/caronte/parsers"
 	log "github.com/sirupsen/logrus"
 	"time"
 )
@@ -25,13 +27,14 @@ type ConnectionStream struct {
 type PatternSlice [2]uint64
 
 type Payload struct {
-	FromClient      bool         `json:"from_client"`
-	Content         string       `json:"content"`
-	DecodedContent  string       `json:"decoded_content"`
-	Index           int          `json:"index"`
-	Timestamp       time.Time    `json:"timestamp"`
-	IsRetransmitted bool         `json:"is_retransmitted"`
-	RegexMatches    []RegexSlice `json:"regex_matches"`
+	FromClient             bool             `json:"from_client"`
+	Content                string           `json:"content"`
+	Metadata               parsers.Metadata `json:"metadata"`
+	IsMetadataContinuation bool             `json:"is_metadata_continuation"`
+	Index                  int              `json:"index"`
+	Timestamp              time.Time        `json:"timestamp"`
+	IsRetransmitted        bool             `json:"is_retransmitted"`
+	RegexMatches           []RegexSlice     `json:"regex_matches"`
 }
 
 type RegexSlice struct {
@@ -56,8 +59,8 @@ func NewConnectionStreamsController(storage Storage) ConnectionStreamsController
 }
 
 func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, connectionID RowID,
-	format QueryFormat) []Payload {
-	payloads := make([]Payload, 0, InitialPayloadsSize)
+	format QueryFormat) []*Payload {
+	payloads := make([]*Payload, 0, InitialPayloadsSize)
 	var clientIndex, serverIndex, globalIndex uint64
 
 	if format.Limit <= 0 {
@@ -76,7 +79,11 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c
 		return serverBlocksIndex < len(serverStream.BlocksIndexes)
 	}
 
-	var payload Payload
+	var payload *Payload
+	payloadsBuffer := make([]*Payload, 0, 16)
+	contentChunkBuffer := new(bytes.Buffer)
+	var lastContentSlice []byte
+	var sideChanged, lastClient, lastServer bool
 	for !clientStream.ID.IsZero() || !serverStream.ID.IsZero() {
 		if hasClientBlocks() && (!hasServerBlocks() || // next payload is from client
 			clientStream.BlocksTimestamps[clientBlocksIndex].UnixNano() <=
@@ -86,14 +93,13 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c
 			if clientBlocksIndex < len(clientStream.BlocksIndexes)-1 {
 				end = clientStream.BlocksIndexes[clientBlocksIndex+1]
 			} else {
-				end = len(clientStream.Payload) - 1
+				end = len(clientStream.Payload)
 			}
 			size := uint64(end - start)
 
-			payload = Payload{
+			payload = &Payload{
 				FromClient:      true,
 				Content:         DecodeBytes(clientStream.Payload[start:end], format.Format),
-				//Request:		 ReadRequest(content),
 				Index:           start,
 				Timestamp:       clientStream.BlocksTimestamps[clientBlocksIndex],
 				IsRetransmitted: clientStream.BlocksLoss[clientBlocksIndex],
@@ -102,25 +108,22 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c
 			clientIndex += size
 			globalIndex += size
 			clientBlocksIndex++
+
+			lastContentSlice = clientStream.Payload[start:end]
+			sideChanged, lastClient, lastServer = lastServer, true, false
 		} else { // next payload is from server
 			start := serverStream.BlocksIndexes[serverBlocksIndex]
 			end := 0
 			if serverBlocksIndex < len(serverStream.BlocksIndexes)-1 {
 				end = serverStream.BlocksIndexes[serverBlocksIndex+1]
 			} else {
-				end = len(serverStream.Payload) - 1
+				end = len(serverStream.Payload)
 			}
 			size := uint64(end - start)
 
-			content := DecodeBytes(serverStream.Payload[start:end], format.Format)
-
-			plainContent := DecodeBytes(serverStream.Payload[start:end], "default")
-			decodedContent := DecodeBytes([]byte(DecodeHttpResponse(plainContent)), format.Format)
-
-			payload = Payload{
+			payload = &Payload{
 				FromClient:      false,
-				Content:         content,
-				DecodedContent:  decodedContent,
+				Content:         DecodeBytes(serverStream.Payload[start:end], format.Format),
 				Index:           start,
 				Timestamp:       serverStream.BlocksTimestamps[serverBlocksIndex],
 				IsRetransmitted: serverStream.BlocksLoss[serverBlocksIndex],
@@ -129,13 +132,9 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c
 			serverIndex += size
 			globalIndex += size
 			serverBlocksIndex++
-		}
 
-		if globalIndex > format.Skip {
-			payloads = append(payloads, payload)
-		}
-		if globalIndex > format.Skip+format.Limit {
-			return payloads
+			lastContentSlice = serverStream.Payload[start:end]
+			sideChanged, lastClient, lastServer = lastClient, false, true
 		}
 
 		if !hasClientBlocks() {
@@ -148,6 +147,39 @@ func (csc ConnectionStreamsController) GetConnectionPayload(c context.Context, c
 			serverBlocksIndex = 0
 			serverStream = csc.getConnectionStream(c, connectionID, false, serverDocumentIndex)
 		}
+
+		updateMetadata := func() {
+			metadata := parsers.Parse(contentChunkBuffer.Bytes())
+			var isMetadataContinuation bool
+			for _, elem := range payloadsBuffer {
+				elem.Metadata = metadata
+				elem.IsMetadataContinuation = isMetadataContinuation
+				isMetadataContinuation = true
+			}
+
+			payloadsBuffer = payloadsBuffer[:0]
+			contentChunkBuffer.Reset()
+		}
+
+		if sideChanged {
+			updateMetadata()
+		}
+		payloadsBuffer = append(payloadsBuffer, payload)
+		contentChunkBuffer.Write(lastContentSlice)
+
+		if clientStream.ID.IsZero() && serverStream.ID.IsZero() {
+			updateMetadata()
+		}
+
+		if globalIndex > format.Skip {
+			// problem: waste of time if the payload is discarded
+			payloads = append(payloads, payload)
+		}
+		if globalIndex > format.Skip+format.Limit {
+			// problem: the last chunk is not parsed, but can be ok because it is not finished
+			updateMetadata()
+			return payloads
+		}
 	}
 
 	return payloads
diff --git a/frontend/src/components/Connection.js b/frontend/src/components/Connection.js
index e41f542..93c6438 100644
--- a/frontend/src/components/Connection.js
+++ b/frontend/src/components/Connection.js
@@ -57,6 +57,11 @@ class Connection extends Component {
         let closedAt = new Date(conn.closed_at);
         let processedAt = new Date(conn.processed_at);
         let duration = ((closedAt - startedAt) / 1000).toFixed(3);
+        if (duration > 1000 || duration < -1000) {
+            duration = "∞";
+        } else {
+            duration += "s";
+        }
         let timeInfo = <div>
             <span>Started at {startedAt.toLocaleDateString() + " " + startedAt.toLocaleTimeString()}</span><br/>
             <span>Processed at {processedAt.toLocaleDateString() + " " + processedAt.toLocaleTimeString()}</span><br/>
@@ -106,7 +111,7 @@ class Connection extends Component {
                 <td className="clickable" onClick={this.props.onSelected}>
                     <OverlayTrigger trigger={["focus", "hover"]} placement="right"
                                     overlay={popoverFor("duration", timeInfo)}>
-                        <span className="test-tooltip">{duration}s</span>
+                        <span className="test-tooltip">{duration}</span>
                     </OverlayTrigger>
                 </td>
                 <td className="clickable" onClick={this.props.onSelected}>{conn.client_bytes}</td>
diff --git a/frontend/src/components/ConnectionContent.js b/frontend/src/components/ConnectionContent.js
index 2100a68..20ec92b 100644
--- a/frontend/src/components/ConnectionContent.js
+++ b/frontend/src/components/ConnectionContent.js
@@ -1,7 +1,10 @@
 import React, {Component} from 'react';
 import './ConnectionContent.scss';
-import {Dropdown, Button} from 'react-bootstrap';
+import {Button, Dropdown, Row} from 'react-bootstrap';
 import axios from 'axios';
+import MessageAction from "./MessageAction";
+
+const classNames = require('classnames');
 
 class ConnectionContent extends Component {
 
@@ -11,7 +14,8 @@ class ConnectionContent extends Component {
             loading: false,
             connectionContent: null,
             format: "default",
-            decoded: false,
+            tryParse: true,
+            messageActionDialog: null
         };
 
         this.validFormats = ["default", "hex", "hexdump", "base32", "base64", "ascii", "binary", "decimal", "octal"];
@@ -38,66 +42,148 @@ class ConnectionContent extends Component {
         }
     }
 
-    toggleDecoded() {
-		this.setState({decoded: !this.state.decoded});
+    tryParseConnectionMessage(connectionMessage) {
+        if (connectionMessage.metadata == null) {
+            return connectionMessage.content;
+        }
+        if (connectionMessage["is_metadata_continuation"]) {
+            return <span style={{"fontSize": "12px"}}>**already parsed in previous messages**</span>;
+        }
+
+        let unrollMap = (obj) => obj == null ? null : Object.entries(obj).map(([key, value]) =>
+            <p><strong>{key}</strong>: {value}</p>
+        );
+
+        let m = connectionMessage.metadata;
+        switch (m.type) {
+            case "http-request":
+                let url = <i><u><a href={"http://" + m.host + m.url} target="_blank"
+                                   rel="noopener noreferrer">{m.host}{m.url}</a></u></i>;
+                return <span className="type-http-request">
+                    <p style={{"marginBottom": "7px"}}><strong>{m.method}</strong> {url} {m.protocol}</p>
+                    {unrollMap(m.headers)}
+                    <div style={{"margin": "20px 0"}}>{m.body}</div>
+                    {unrollMap(m.trailers)}
+                </span>;
+            case "http-response":
+                return <span className="type-http-response">
+                    <p style={{"marginBottom": "7px"}}>{m.protocol} <strong>{m.status}</strong></p>
+                    {unrollMap(m.headers)}
+                    <div style={{"margin": "20px 0"}}>{m.body}</div>
+                    {unrollMap(m.trailers)}
+                </span>;
+            default:
+                return connectionMessage.content;
+        }
+    }
+
+    connectionsActions(connectionMessage) {
+        if (connectionMessage.metadata == null || connectionMessage.metadata["reproducers"] === undefined) {
+            return null;
+        }
+
+        return Object.entries(connectionMessage.metadata["reproducers"]).map(([actionName, actionValue]) =>
+            <Button size="sm" key={actionName + "_button"} onClick={() => {
+                this.setState({
+                    messageActionDialog: <MessageAction actionName={actionName} actionValue={actionValue}
+                                                        onHide={() => this.setState({messageActionDialog: null})}/>
+                });
+            }}>{actionName}</Button>
+        );
     }
 
     render() {
         let content = this.state.connectionContent;
 
-        if (content === null) {
-            return <div>nope</div>;
+        if (content == null) {
+            return <div>select a connection to view</div>;
         }
 
         let payload = content.map((c, i) =>
-            <span key={`content-${i}`} className={c.from_client ? "from-client" : "from-server"}>
-				{c.from_client
-				?
-				<div id="content">{c.content}</div>
-				:
-					<>
-					{c.decoded_content
-					?
-					<>
-					<div style={{display: this.state.decoded ? 'none':'inherit'}} id="content">{c.content}</div>
-					<div style={{display: this.state.decoded ? 'inherit':'none'}} id="decoded_content">{c.decoded_content}</div>
-					</>
-					:
-					<div id="content">{c.content}</div>
-					}
-					</>
-				}
-            </span>
+            <div key={`content-${i}`}
+                 className={classNames("connection-message", c.from_client ? "from-client" : "from-server")}>
+                <div className="connection-message-header container-fluid">
+                    <div className="row">
+                        <div className="connection-message-info col">
+                            <span><strong>offset</strong>: {c.index}</span> | <span><strong>timestamp</strong>: {c.timestamp}
+                        </span> | <span><strong>retransmitted</strong>: {c["is_retransmitted"] ? "yes" : "no"}</span>
+                        </div>
+                        <div className="connection-message-actions col-auto">{this.connectionsActions(c)}</div>
+                    </div>
+                </div>
+                <div className="connection-message-label">{c.from_client ? "client" : "server"}</div>
+                <div
+                    className={classNames("message-content", this.state.decoded ? "message-parsed" : "message-original")}>
+                    {this.state.tryParse && this.state.format === "default" ? this.tryParseConnectionMessage(c) : c.content}
+                </div>
+            </div>
         );
 
         return (
             <div className="connection-content">
-                <div className="connection-content-options">
-                    <Dropdown onSelect={this.setFormat} >
-                        <Dropdown.Toggle size="sm" id="dropdown-basic">
-                            format
-                        </Dropdown.Toggle>
-
-                        <Dropdown.Menu>
-                            <Dropdown.Item eventKey="default" active={this.state.format === "default"}>plain</Dropdown.Item>
-                            <Dropdown.Item eventKey="hex" active={this.state.format === "hex"}>hex</Dropdown.Item>
-                            <Dropdown.Item eventKey="hexdump" active={this.state.format === "hexdump"}>hexdump</Dropdown.Item>
-                            <Dropdown.Item eventKey="base32" active={this.state.format === "base32"}>base32</Dropdown.Item>
-                            <Dropdown.Item eventKey="base64" active={this.state.format === "base64"}>base64</Dropdown.Item>
-                            <Dropdown.Item eventKey="ascii" active={this.state.format === "ascii"}>ascii</Dropdown.Item>
-                            <Dropdown.Item eventKey="binary" active={this.state.format === "binary"}>binary</Dropdown.Item>
-                            <Dropdown.Item eventKey="decimal" active={this.state.format === "decimal"}>decimal</Dropdown.Item>
-                            <Dropdown.Item eventKey="octal" active={this.state.format === "octal"}>octal</Dropdown.Item>
-                        </Dropdown.Menu>
-						<Button onClick={() => this.toggleDecoded()}>{this.state.decoded ? "Encode" : "Decode"}</Button>
-
-
-                    </Dropdown>
-
-
+                <div className="connection-content-header container-fluid">
+                    <Row>
+                        <div className="header-info col">
+                            <span><strong>flow</strong>: {this.props.connection.ip_src}:{this.props.connection.port_src} -> {this.props.connection.ip_dst}:{this.props.connection.port_dst}</span>
+                            <span> | <strong>timestamp</strong>: {this.props.connection.started_at}</span>
+                        </div>
+                        <div className="header-actions col-auto">
+                            <Dropdown onSelect={this.setFormat}>
+                                <Dropdown.Toggle size="sm" id="connection-content-format">
+                                    format
+                                </Dropdown.Toggle>
+
+                                <Dropdown.Menu>
+                                    <Dropdown.Item eventKey="default"
+                                                   active={this.state.format === "default"}>plain</Dropdown.Item>
+                                    <Dropdown.Item eventKey="hex"
+                                                   active={this.state.format === "hex"}>hex</Dropdown.Item>
+                                    <Dropdown.Item eventKey="hexdump"
+                                                   active={this.state.format === "hexdump"}>hexdump</Dropdown.Item>
+                                    <Dropdown.Item eventKey="base32"
+                                                   active={this.state.format === "base32"}>base32</Dropdown.Item>
+                                    <Dropdown.Item eventKey="base64"
+                                                   active={this.state.format === "base64"}>base64</Dropdown.Item>
+                                    <Dropdown.Item eventKey="ascii"
+                                                   active={this.state.format === "ascii"}>ascii</Dropdown.Item>
+                                    <Dropdown.Item eventKey="binary"
+                                                   active={this.state.format === "binary"}>binary</Dropdown.Item>
+                                    <Dropdown.Item eventKey="decimal"
+                                                   active={this.state.format === "decimal"}>decimal</Dropdown.Item>
+                                    <Dropdown.Item eventKey="octal"
+                                                   active={this.state.format === "octal"}>octal</Dropdown.Item>
+                                </Dropdown.Menu>
+                            </Dropdown>
+
+                            <Dropdown>
+                                <Dropdown.Toggle size="sm" id="connection-content-view">
+                                    view_as
+                                </Dropdown.Toggle>
+
+                                <Dropdown.Menu>
+                                    <Dropdown.Item eventKey="default" active={true}>default</Dropdown.Item>
+                                </Dropdown.Menu>
+
+                            </Dropdown>
+
+                            <Dropdown>
+                                <Dropdown.Toggle size="sm" id="connection-content-download">
+                                    download_as
+                                </Dropdown.Toggle>
+
+                                <Dropdown.Menu>
+                                    <Dropdown.Item eventKey="nl_separated">nl_separated</Dropdown.Item>
+                                    <Dropdown.Item eventKey="only_client">only_client</Dropdown.Item>
+                                    <Dropdown.Item eventKey="only_server">only_server</Dropdown.Item>
+                                </Dropdown.Menu>
+
+                            </Dropdown>
+                        </div>
+                    </Row>
                 </div>
 
                 <pre>{payload}</pre>
+                {this.state.messageActionDialog}
             </div>
         );
     }
diff --git a/frontend/src/components/ConnectionContent.scss b/frontend/src/components/ConnectionContent.scss
index 5a17066..8ee31ec 100644
--- a/frontend/src/components/ConnectionContent.scss
+++ b/frontend/src/components/ConnectionContent.scss
@@ -1,29 +1,108 @@
 @import '../colors.scss';
 
 .connection-content {
-  background-color: $color-primary-3;
+  background-color: $color-primary-0;
+  padding: 10px 10px 0;
   height: 100%;
-  overflow: fixed;
 
   pre {
-    background-color: $color-primary-0;
-    padding: 10px 20px;
     word-break: break-word;
-    max-width: 100%;
     white-space: pre-wrap;
-    height: 95%;
-  }
+    overflow-x: hidden;
+    height: calc(100% - 31px);
+    padding: 0 10px;
 
-  .from-client {
-    color: #d4e0fc;
+    p {
+      margin: 0;
+      padding: 0;
+    }
   }
 
-  .from-server {
-    color: $color-secondary-4;
+  .connection-message {
+    border: 4px solid $color-primary-3;
+    border-top: 0;
+    margin: 10px 0;
+    position: relative;
+
+    .connection-message-header {
+      background-color: $color-primary-3;
+      height: 25px;
+
+      .connection-message-info {
+        font-size: 11px;
+        margin-left: -10px;
+        margin-top: 6px;
+      }
+
+      .connection-message-actions {
+        margin-right: -18px;
+        display: none;
+
+        button {
+          margin: 0 3px;
+          font-size: 11px;
+          padding: 5px;
+        }
+      }
+    }
+
+    .message-content {
+      padding: 10px;
+    }
 
-    &:hover {
+    &:hover .connection-message-actions {
+      display: block;
+    }
+
+    .connection-message-label {
+      position: absolute;
       background-color: $color-primary-3;
-      border-top: 1px solid $color-primary-1;
+      top: 0;
+      padding: 10px 0;
+      font-size: 12px;
+
+      writing-mode: vertical-rl;
+      text-orientation: mixed;
+    }
+
+    &.from-client {
+      color: $color-primary-4;
+      margin-right: 100px;
+
+      .connection-message-label {
+        right: -22px;
+      }
+    }
+
+    &.from-server {
+      color: $color-primary-4;
+      margin-left: 100px;
+
+      .connection-message-label {
+        left: -22px;
+        transform: rotate(-180deg);
+      }
+    }
+
+  }
+
+  .connection-content-header {
+    background-color: $color-primary-2;
+    padding: 0;
+    height: 31px;
+
+    .header-info {
+      padding-top: 5px;
+      padding-left: 20px;
+      font-size: 12px;
+    }
+
+    .header-actions {
+      .dropdown {
+        display: inline-block;
+      }
     }
   }
+
+
 }
diff --git a/frontend/src/components/MessageAction.js b/frontend/src/components/MessageAction.js
new file mode 100644
index 0000000..2c85d84
--- /dev/null
+++ b/frontend/src/components/MessageAction.js
@@ -0,0 +1,52 @@
+import React, {Component} from 'react';
+import './MessageAction.scss';
+import {Button, FormControl, InputGroup, Modal} from "react-bootstrap";
+
+class MessageAction extends Component {
+
+    constructor(props) {
+        super(props);
+        this.state = {
+            copyButtonText: "copy"
+        };
+        this.actionValue = React.createRef();
+        this.copyActionValue = this.copyActionValue.bind(this);
+    }
+
+    copyActionValue() {
+        this.actionValue.current.select();
+        document.execCommand('copy');
+        this.setState({copyButtonText: "copied!"});
+        setTimeout(() => this.setState({copyButtonText: "copy"}), 3000);
+    }
+
+    render() {
+        return (
+            <Modal
+                {...this.props}
+                show="true"
+                size="lg"
+                aria-labelledby="message-action-dialog"
+                centered
+            >
+                <Modal.Header>
+                    <Modal.Title id="message-action-dialog">
+                        {this.props.actionName}
+                    </Modal.Title>
+                </Modal.Header>
+                <Modal.Body>
+                    <InputGroup>
+                        <FormControl as="textarea" className="message-action-value" readOnly={true}
+                                     style={{"height": "300px"}} value={this.props.actionValue} ref={this.actionValue} />
+                    </InputGroup>
+                </Modal.Body>
+                <Modal.Footer className="dialog-footer">
+                    <Button variant="green" onClick={this.copyActionValue}>{this.state.copyButtonText}</Button>
+                    <Button variant="red" onClick={this.props.onHide}>close</Button>
+                </Modal.Footer>
+            </Modal>
+        );
+    }
+}
+
+export default MessageAction;
diff --git a/frontend/src/components/MessageAction.scss b/frontend/src/components/MessageAction.scss
new file mode 100644
index 0000000..f3a8772
--- /dev/null
+++ b/frontend/src/components/MessageAction.scss
@@ -0,0 +1,8 @@
+@import '../colors.scss';
+
+.message-action-value {
+  font-size: 13px;
+  padding: 15px;
+  background-color: $color-primary-2;
+  color: $color-primary-4;
+}
\ No newline at end of file
diff --git a/frontend/src/utils.js b/frontend/src/utils.js
index 26c10d3..7381f69 100644
--- a/frontend/src/utils.js
+++ b/frontend/src/utils.js
@@ -59,3 +59,8 @@ export function timestampToTime(timestamp) {
     let seconds = "0" + d.getSeconds();
     return hours + ':' + minutes.substr(-2) + ':' + seconds.substr(-2);
 }
+
+export function timestampToDateTime(timestamp) {
+    let d = new Date(timestamp);
+    return d.toLocaleDateString() + " " + d.toLocaleTimeString();
+}
diff --git a/go.mod b/go.mod
index 1281ae8..308b16b 100644
--- a/go.mod
+++ b/go.mod
@@ -17,4 +17,5 @@ require (
 	go.mongodb.org/mongo-driver v1.3.1
 	golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
 	golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect
+	moul.io/http2curl v1.0.0
 )
diff --git a/go.sum b/go.sum
index d17dea6..fd63c39 100644
--- a/go.sum
+++ b/go.sum
@@ -179,3 +179,5 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
+moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
diff --git a/parsers/http_request_parser.go b/parsers/http_request_parser.go
new file mode 100644
index 0000000..e2224b8
--- /dev/null
+++ b/parsers/http_request_parser.go
@@ -0,0 +1,149 @@
+package parsers
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+	"moul.io/http2curl"
+	"net/http"
+	"strings"
+)
+
+type HttpRequestMetadata struct {
+	BasicMetadata
+	Method        string                         `json:"method"`
+	URL           string                         `json:"url"`
+	Protocol      string                         `json:"protocol"`
+	Host          string                         `json:"host"`
+	Headers       map[string]string              `json:"headers"`
+	Cookies       map[string]string              `json:"cookies" binding:"omitempty"`
+	ContentLength int64                          `json:"content_length"`
+	FormData      map[string]string              `json:"form_data" binding:"omitempty"`
+	Body          string                         `json:"body" binding:"omitempty"`
+	Trailer       map[string]string              `json:"trailer" binding:"omitempty"`
+	Reproducers   HttpRequestMetadataReproducers `json:"reproducers"`
+}
+
+type HttpRequestMetadataReproducers struct {
+	CurlCommand  string `json:"curl_command"`
+	RequestsCode string `json:"requests_code"`
+	FetchRequest string `json:"fetch_request"`
+}
+
+type HttpRequestParser struct {
+}
+
+func (p HttpRequestParser) TryParse(content []byte) Metadata {
+	reader := bufio.NewReader(bytes.NewReader(content))
+	request, err := http.ReadRequest(reader)
+	if err != nil {
+		return nil
+	}
+	var body string
+	if buffer, err := ioutil.ReadAll(request.Body); err == nil {
+		body = string(buffer)
+	} else {
+		log.WithError(err).Error("failed to read body in http_request_parser")
+		return nil
+	}
+	_ = request.Body.Close()
+	_ = request.ParseForm()
+
+	return HttpRequestMetadata{
+		BasicMetadata: BasicMetadata{"http-request"},
+		Method:        request.Method,
+		URL:           request.URL.String(),
+		Protocol:      request.Proto,
+		Host:          request.Host,
+		Headers:       JoinArrayMap(request.Header),
+		Cookies:       CookiesMap(request.Cookies()),
+		ContentLength: request.ContentLength,
+		FormData:      JoinArrayMap(request.Form),
+		Body:          body,
+		Trailer:       JoinArrayMap(request.Trailer),
+		Reproducers: HttpRequestMetadataReproducers{
+			CurlCommand:  curlCommand(content),
+			RequestsCode: requestsCode(request),
+			FetchRequest: fetchRequest(request, body),
+		},
+	}
+}
+
+func curlCommand(content []byte) string {
+	// a new reader is required because all the body is read before and GetBody() doesn't works
+	reader := bufio.NewReader(bytes.NewReader(content))
+	request, _ := http.ReadRequest(reader)
+	if command, err := http2curl.GetCurlCommand(request); err == nil {
+		return command.String()
+	} else {
+		return err.Error()
+	}
+}
+
+func requestsCode(request *http.Request) string {
+	var b strings.Builder
+	var params string
+	if request.Form != nil {
+		params = toJson(JoinArrayMap(request.PostForm))
+	}
+	headers := toJson(JoinArrayMap(request.Header))
+	cookies := toJson(CookiesMap(request.Cookies()))
+
+	b.WriteString("import requests\n\nresponse = requests." + strings.ToLower(request.Method) + "(")
+	b.WriteString("\"" + request.URL.String() + "\"")
+	if params != "" {
+		b.WriteString(", data = " + params)
+	}
+	if headers != "" {
+		b.WriteString(", headers = " + headers)
+	}
+	if cookies != "" {
+		b.WriteString(", cookies = " + cookies)
+	}
+	b.WriteString(")\n")
+	b.WriteString(`
+# print(response.url)
+# print(response.text)
+# print(response.content)
+# print(response.json())
+# print(response.raw)
+# print(response.status_code)
+# print(response.cookies)
+# print(response.history)
+`)
+
+	return b.String()
+}
+
+func fetchRequest(request *http.Request, body string) string {
+	headers := JoinArrayMap(request.Header)
+	data := make(map[string]interface{})
+	data["headers"] = headers
+	if referrer := request.Header.Get("referrer"); referrer != "" {
+		data["Referrer"] = referrer
+	}
+	// TODO: referrerPolicy
+	if body == "" {
+		data["body"] = nil
+	} else {
+		data["body"] = body
+	}
+	data["method"] = request.Method
+	// TODO: mode
+
+	if jsonData := toJson(data); jsonData != "" {
+		return "fetch(\"" + request.URL.String() + "\", " + jsonData + ");"
+	} else {
+		return "invalid-request"
+	}
+}
+
+func toJson(obj interface{}) string {
+	if buffer, err := json.MarshalIndent(obj, "", "\t"); err == nil {
+		return string(buffer)
+	} else {
+		return ""
+	}
+}
diff --git a/parsers/http_response_parser.go b/parsers/http_response_parser.go
new file mode 100644
index 0000000..1770116
--- /dev/null
+++ b/parsers/http_response_parser.go
@@ -0,0 +1,77 @@
+package parsers
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+	"net/http"
+)
+
+type HttpResponseMetadata struct {
+	BasicMetadata
+	Status           string            `json:"status"`
+	StatusCode       int               `json:"status_code"`
+	Protocol         string            `json:"protocol"`
+	Headers          map[string]string `json:"headers"`
+	ConnectionClosed bool              `json:"connection_closed"`
+	Cookies          map[string]string `json:"cookies" binding:"omitempty"`
+	Location         string            `json:"location" binding:"omitempty"`
+	Compressed       bool              `json:"compressed"`
+	Body             string            `json:"body" binding:"omitempty"`
+	Trailer          map[string]string `json:"trailer" binding:"omitempty"`
+}
+
+type HttpResponseParser struct {
+}
+
+func (p HttpResponseParser) TryParse(content []byte) Metadata {
+	reader := bufio.NewReader(bytes.NewReader(content))
+	response, err := http.ReadResponse(reader, nil)
+	if err != nil {
+		return nil
+	}
+	var body string
+	var compressed bool
+	switch response.Header.Get("Content-Encoding") {
+	case "gzip":
+		if gzipReader, err := gzip.NewReader(response.Body); err == nil {
+			if buffer, err := ioutil.ReadAll(gzipReader); err == nil {
+				body = string(buffer)
+				compressed = true
+			} else {
+				log.WithError(err).Error("failed to read gzipped body in http_response_parser")
+				return nil
+			}
+			_ = gzipReader.Close()
+		}
+	default:
+		if buffer, err := ioutil.ReadAll(response.Body); err == nil {
+			body = string(buffer)
+		} else {
+			log.WithError(err).Error("failed to read body in http_response_parser")
+			return nil
+		}
+	}
+	_ = response.Body.Close()
+
+	var location string
+	if locationUrl, err := response.Location(); err == nil {
+		location = locationUrl.String()
+	}
+
+	return HttpResponseMetadata{
+		BasicMetadata:    BasicMetadata{"http-response"},
+		Status:           response.Status,
+		StatusCode:       response.StatusCode,
+		Protocol:         response.Proto,
+		Headers:          JoinArrayMap(response.Header),
+		ConnectionClosed: response.Close,
+		Cookies:          CookiesMap(response.Cookies()),
+		Location:         location,
+		Compressed:       compressed,
+		Body:             body,
+		Trailer:          JoinArrayMap(response.Trailer),
+	}
+}
diff --git a/parsers/parser.go b/parsers/parser.go
new file mode 100644
index 0000000..06cc0dc
--- /dev/null
+++ b/parsers/parser.go
@@ -0,0 +1,28 @@
+package parsers
+
+type Parser interface {
+	TryParse(content []byte) Metadata
+
+}
+
+type Metadata interface {
+}
+
+type BasicMetadata struct {
+	Type string `json:"type"`
+}
+
+var parsers = []Parser{	// order matter
+	HttpRequestParser{},
+	HttpResponseParser{},
+}
+
+func Parse(content []byte) Metadata {
+	for _, parser := range parsers {
+		if metadata := parser.TryParse(content); metadata != nil {
+			return metadata
+		}
+	}
+
+	return nil
+}
diff --git a/parsers/parser_utils.go b/parsers/parser_utils.go
new file mode 100644
index 0000000..b688262
--- /dev/null
+++ b/parsers/parser_utils.go
@@ -0,0 +1,24 @@
+package parsers
+
+import (
+	"net/http"
+	"strings"
+)
+
+func JoinArrayMap(obj map[string][]string) map[string]string {
+	headers := make(map[string]string, len(obj))
+	for key, value := range obj {
+		headers[key] = strings.Join(value, ";")
+	}
+
+	return headers
+}
+
+func CookiesMap(cookiesArray []*http.Cookie) map[string]string {
+	cookies := make(map[string]string, len(cookiesArray))
+	for _, cookie := range cookiesArray {
+		cookies[cookie.Name] = cookie.Value
+	}
+
+	return cookies
+}
diff --git a/utils.go b/utils.go
index b07244d..a14fdca 100644
--- a/utils.go
+++ b/utils.go
@@ -13,11 +13,6 @@ import (
 	"net"
 	"os"
 	"time"
-	"net/http"
-	"bufio"
-	"strings"
-	"io/ioutil"
-	"compress/gzip"
 )
 
 func Sha256Sum(fileName string) (string, error) {
@@ -113,83 +108,6 @@ func DecodeBytes(buffer []byte, format string) string {
 	}
 }
 
-func ReadRequest(raw string) http.Request {
-	reader := bufio.NewReader(strings.NewReader(raw))
-	req,err := http.ReadRequest(reader)
-	if err != nil{
-		log.Info("Reading request: ",req)
-		return http.Request{}
-	}
-	return *req
-}
-
-func GetHeader(raw string) string{
-	tmp := strings.Split(raw,"\r\n")
-	end := len(tmp)
-	for i, line := range tmp{
-		if line == ""{
-			end = i
-			break
-		}
-	}
-	return strings.Join(tmp[:end],"\r\n")
-}
-
-func GetBody(raw string) string{
-	tmp := strings.Split(raw,"\r\n")
-	start := 0
-	for i, line := range tmp{
-		if line == ""{
-			start = i + 2
-			break
-		}
-	}
-	return strings.Join(tmp[start:],"\r\n")
-}
-
-func DecodeHttpResponse(raw string) string {
-	body := []byte{}
-	reader := bufio.NewReader(strings.NewReader(raw))
-	resp,err := http.ReadResponse(reader, &http.Request{})
-	if err != nil{
-		log.Info("Reading response: ",resp)
-		return ""
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
-		var bodyReader io.ReadCloser
-		switch resp.Header.Get("Content-Encoding") {
-		case "gzip":
-			bodyReader, err = gzip.NewReader(resp.Body)
-			if err != nil {
-				log.Error("Gunzipping body: ",err)
-			}
-			defer bodyReader.Close()
-			body, err = ioutil.ReadAll(bodyReader)
-			if err != nil{
-				log.Error("Reading gzipped body: ",err)
-				// if the response is malformed
-				// or the connection is closed
-				fallbackReader, _ := gzip.NewReader(strings.NewReader(GetBody(raw)))
-				body, err = ioutil.ReadAll(fallbackReader)
-				if err != nil{
-					log.Error(string(body))
-				}
-			}
-		default:
-			bodyReader = resp.Body
-			body, err = ioutil.ReadAll(bodyReader)
-			if err != nil{
-				log.Error("Reading body: ",err)
-				body = []byte(GetBody(raw))
-			}
-		}
-	}
-	return GetHeader(raw) + "\r\n\r\n"+ string(body)
-}
-
 func CopyFile(dst, src string) error {
 	in, err := os.Open(src)
 	if err != nil {