Before storage refactor

13 files changed, 590 insertions, 54 deletions

diff --git a/Dockerfile b/Dockerfile
index 0c6f4f7..4f90ef8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM eciavatta/caronte-env
+FROM eciavatta/caronte-env:latest
 
 COPY . /caronte
 
diff --git a/README.md b/README.md
index 9fc1fbb..50e099e 100644
--- a/README.md
+++ b/README.md
@@ -2,11 +2,10 @@
 
 [![Build Status](https://travis-ci.com/eciavatta/caronte.svg?branch=develop)](https://travis-ci.com/eciavatta/caronte)
 [![codecov](https://codecov.io/gh/eciavatta/caronte/branch/develop/graph/badge.svg)](https://codecov.io/gh/eciavatta/caronte)
-[![GPL License][license-shield]][license-url]
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/009dca44f4da4118a20aed2b9b7610c0)](https://www.codacy.com/manual/eciavatta/caronte?utm_source=github.com&utm_medium=referral&utm_content=eciavatta/caronte&utm_campaign=Badge_Grade)
 
 <img align="left" src="https://divinacommedia.weebly.com/uploads/5/5/2/3/5523249/1299707879.jpg">
 Caronte is a tool to analyze the network flow during capture the flag events of type attack/defence.
 It reassembles TCP packets captured in pcap files to rebuild TCP connections, and analyzes each connection to find user-defined patterns.
 The patterns can be defined as regex or using protocol specific rules.
 The connection flows are saved into a database and can be visualized with the web application. REST API are also provided.
-
diff --git a/caronte.go b/caronte.go
index 6b33318..a32b619 100644
--- a/caronte.go
+++ b/caronte.go
@@ -1,26 +1,33 @@
 package main
 
 import (
+	"flag"
 	"fmt"
-	"net"
+	"github.com/gin-gonic/gin"
+	"log"
 )
 
 func main() {
-	// pattern.Flags |= hyperscan.SomLeftMost
+	// test(); return
+	mongoHost := flag.String("mongo-host", "localhost", "address of MongoDB")
+	mongoPort := flag.Int("mongo-port", 27017, "port of MongoDB")
+	dbName := flag.String("db-name", "caronte", "name of database to use")
 
-	storage := NewMongoStorage("localhost", 27017, "testing")
+	bindAddress := flag.String("bind-address", "0.0.0.0", "address where server is bind")
+	bindPort := flag.Int("bind-port", 3333, "port where server is bind")
+
+	flag.Parse()
+
+	storage := NewMongoStorage(*mongoHost, *mongoPort, *dbName)
 	err := storage.Connect(nil)
 	if err != nil {
-		panic(err)
+		log.Panicln("failed to connect to MongoDB:", err)
 	}
 
-	importer := NewPcapImporter(storage, net.ParseIP("10.10.10.10"))
-
-	sessionId, err := importer.ImportPcap("capture_00459_20190627165500.pcap")
+	router := gin.Default()
+	ApplicationRoutes(router)
+	err = router.Run(fmt.Sprintf("%s:%v", *bindAddress, *bindPort))
 	if err != nil {
-		fmt.Println(err)
-	} else {
-		fmt.Println(sessionId)
+		log.Panicln("failed to create the server:", err)
 	}
-
 }
diff --git a/caronte_test.go b/caronte_test.go
index cbf867d..9942086 100644
--- a/caronte_test.go
+++ b/caronte_test.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"log"
 	"os"
 	"testing"
 	"time"
@@ -14,6 +15,9 @@ import (
 var storage Storage
 var testContext context.Context
 
+const testInsertManyFindCollection = "testFi"
+const testCollection = "characters"
+
 func TestMain(m *testing.M) {
 	mongoHost, ok := os.LookupEnv("MONGO_HOST")
 	if !ok {
@@ -31,15 +35,20 @@ func TestMain(m *testing.M) {
 		panic("failed to create mongo client")
 	}
 
-	db := client.Database(fmt.Sprintf("%x", uniqueDatabaseName[:31]))
+	dbName := fmt.Sprintf("%x", uniqueDatabaseName[:31])
+	db := client.Database(dbName)
+	log.Println("using database", dbName)
 	mongoStorage := MongoStorage{
 		client:      client,
-		collections: map[string]*mongo.Collection{testCollection: db.Collection(testCollection)},
+		collections: map[string]*mongo.Collection{
+			testInsertManyFindCollection: db.Collection(testInsertManyFindCollection),
+			testCollection: db.Collection(testCollection),
+		},
 	}
 
 	testContext, _ = context.WithTimeout(context.Background(), 10 * time.Second)
 
-	err = mongoStorage.Connect(nil)
+	err = mongoStorage.Connect(testContext)
 	if err != nil {
 		panic(err)
 	}
diff --git a/docker-compose.testing.yml b/docker-compose.testing.yml
index 51997d6..ff33e3c 100644
--- a/docker-compose.testing.yml
+++ b/docker-compose.testing.yml
@@ -18,7 +18,7 @@ services:
       - mongo
     networks:
       - acheronet
-    command: "./travis_tests.sh"
+    command: "./scripts/travis_tests.sh"
     environment:
       MONGO_HOST: mongo
       MONGO_PORT: 27017
diff --git a/go.mod b/go.mod
index 66af307..99863d7 100644
--- a/go.mod
+++ b/go.mod
@@ -4,8 +4,14 @@ go 1.14
 
 require (
 	github.com/flier/gohs v1.0.0
+	github.com/gin-gonic/gin v1.6.2
+	github.com/go-playground/validator/v10 v10.2.0
+	github.com/golang/protobuf v1.3.5 // indirect
 	github.com/google/gopacket v1.1.17
-	github.com/stretchr/testify v1.3.0
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/stretchr/testify v1.4.0
 	go.mongodb.org/mongo-driver v1.3.1
 	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
+	golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect
 )
diff --git a/go.sum b/go.sum
index 4d0e509..16a4ce9 100644
--- a/go.sum
+++ b/go.sum
@@ -4,6 +4,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flier/gohs v1.0.0 h1:Q0mmufGWTigzKb140WmJ0+k3EGAf335Qgv/pz5SOPvU=
 github.com/flier/gohs v1.0.0/go.mod h1:Jlg6A1xXSMhPorF74/LkYHkCHZ87Txi8CqIHHyIKgKg=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM=
+github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA=
+github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
+github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
+github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
@@ -30,10 +43,15 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.14/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw=
 github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
 github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
@@ -45,6 +63,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
@@ -57,8 +77,20 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -83,8 +115,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
@@ -112,6 +150,10 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0=
+golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -124,4 +166,10 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
+gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M=
+gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/routes.go b/routes.go
new file mode 100644
index 0000000..f44cff7
--- /dev/null
+++ b/routes.go
@@ -0,0 +1,33 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/go-playground/validator/v10"
+	"log"
+	"net/http"
+)
+
+func ApplicationRoutes(engine *gin.Engine) {
+	engine.Static("/", "./frontend/build")
+
+	api := engine.Group("/api")
+	{
+		api.POST("/rules", func(c *gin.Context) {
+			var rule Rule
+
+			if err := c.ShouldBindJSON(&rule); err != nil {
+				for _, fieldErr := range err.(validator.ValidationErrors) {
+					log.Println(fieldErr)
+					c.JSON(http.StatusBadRequest, gin.H{
+						"error": fmt.Sprintf("field '%v' does not respect the %v(%v) rule",
+							fieldErr.Field(), fieldErr.Tag(), fieldErr.Param()),
+					})
+					return // exit on first error
+				}
+			}
+
+			c.JSON(200, rule)
+		})
+	}
+}
diff --git a/rules_manager.go b/rules_manager.go
new file mode 100644
index 0000000..d6e9aaa
--- /dev/null
+++ b/rules_manager.go
@@ -0,0 +1,242 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/flier/gohs/hyperscan"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"sync"
+	"time"
+)
+
+
+type RegexFlags struct {
+	Caseless bool `json:"caseless"`    // Set case-insensitive matching.
+	DotAll bool `json:"dot_all"`    // Matching a `.` will not exclude newlines.
+	MultiLine bool `json:"multi_line"`  // Set multi-line anchoring.
+	SingleMatch bool `json:"single_match"`  // Set single-match only mode.
+	Utf8Mode bool `json:"utf_8_mode"`         // Enable UTF-8 mode for this expression.
+	UnicodeProperty bool `json:"unicode_property"`  // Enable Unicode property support for this expression
+}
+
+type Pattern struct {
+	Regex string `json:"regex"`
+	Flags RegexFlags `json:"flags"`
+	MinOccurrences int `json:"min_occurrences"`
+	MaxOccurrences int `json:"max_occurrences"`
+	internalId int
+	compiledPattern *hyperscan.Pattern
+}
+
+type Filter struct {
+	ServicePort int
+	ClientAddress string
+	ClientPort int
+	MinDuration int
+	MaxDuration int
+	MinPackets int
+	MaxPackets int
+	MinSize int
+	MaxSize int
+}
+
+type Rule struct {
+	Id string `json:"-" bson:"_id,omitempty"`
+	Name string `json:"name" binding:"required,min=3" bson:"name"`
+	Color string `json:"color" binding:"required,hexcolor" bson:"color"`
+	Notes string `json:"notes" bson:"notes,omitempty"`
+	Enabled bool `json:"enabled" bson:"enabled"`
+	Patterns []Pattern `json:"patterns" binding:"required,min=1" bson:"patterns"`
+	Filter Filter `json:"filter" bson:"filter,omitempty"`
+	Version int64 `json:"version" bson:"version"`
+}
+
+type RulesManager struct {
+	storage         Storage
+	rules           map[string]Rule
+	rulesByName     map[string]Rule
+	ruleIndex       int
+	patterns        map[string]Pattern
+	mPatterns       sync.Mutex
+	databaseUpdated chan interface{}
+}
+
+func NewRulesManager(storage Storage) RulesManager {
+	return RulesManager{
+		storage:        storage,
+		rules:          make(map[string]Rule),
+		patterns:       make(map[string]Pattern),
+		mPatterns:      sync.Mutex{},
+	}
+}
+
+
+
+func (rm RulesManager) LoadRules() error {
+	var rules []Rule
+	if err := rm.storage.Find(nil, Rules, NoFilters, &rules); err != nil {
+		return err
+	}
+
+	var version int64
+	for _, rule := range rules {
+		if err := rm.validateAndAddRuleLocal(&rule); err != nil {
+			log.Printf("failed to import rule %s: %s\n", rule.Name, err)
+			continue
+		}
+		if rule.Version > version {
+			version = rule.Version
+		}
+	}
+
+	rm.ruleIndex = len(rules)
+	return rm.generateDatabase(0)
+}
+
+func (rm RulesManager) AddRule(context context.Context, rule Rule) (string, error) {
+	rm.mPatterns.Lock()
+
+	rule.Id = UniqueKey(time.Now(), uint32(rm.ruleIndex))
+	rule.Enabled = true
+
+	if err := rm.validateAndAddRuleLocal(&rule); err != nil {
+		rm.mPatterns.Unlock()
+		return "", err
+	}
+	rm.mPatterns.Unlock()
+
+	if _, err := rm.storage.InsertOne(context, Rules, rule); err != nil {
+		return "", err
+	}
+
+	return rule.Id, rm.generateDatabase(rule.Id)
+}
+
+
+
+func (rm RulesManager) validateAndAddRuleLocal(rule *Rule) error {
+	if _, alreadyPresent := rm.rulesByName[rule.Name]; alreadyPresent {
+		return errors.New("rule name must be unique")
+	}
+
+	newPatterns := make(map[string]Pattern)
+	for i, pattern := range rule.Patterns {
+		hash := pattern.Hash()
+		if existingPattern, isPresent := rm.patterns[hash]; isPresent {
+			rule.Patterns[i] = existingPattern
+			continue
+		}
+		err := pattern.BuildPattern()
+		if err != nil {
+			return err
+		}
+		pattern.internalId = len(rm.patterns) + len(newPatterns)
+		newPatterns[hash] = pattern
+	}
+
+	for key, value := range newPatterns {
+		rm.patterns[key] = value
+	}
+
+	rm.rules[rule.Id] = *rule
+	rm.rulesByName[rule.Name] = *rule
+
+	return nil
+}
+
+func (rm RulesManager) generateDatabase(version string) error {
+	patterns := make([]*hyperscan.Pattern, len(rm.patterns))
+	for _, pattern := range rm.patterns {
+		patterns = append(patterns, pattern.compiledPattern)
+	}
+	database, err := hyperscan.NewStreamDatabase(patterns...)
+	if err != nil {
+		return err
+	}
+
+	rm.databaseUpdated <- database
+	return nil
+}
+
+
+func (p Pattern) BuildPattern() error {
+	if p.compiledPattern != nil {
+		return nil
+	}
+	if p.MinOccurrences <= 0 {
+		return errors.New("min_occurrences can't be lower than zero")
+	}
+	if p.MaxOccurrences != -1 && p.MinOccurrences < p.MinOccurrences {
+		return errors.New("max_occurrences can't be lower than min_occurrences")
+	}
+
+	hp, err := hyperscan.ParsePattern(fmt.Sprintf("/%s/", p.Regex))
+	if err != nil {
+		return err
+	}
+
+	if p.Flags.Caseless {
+		hp.Flags |= hyperscan.Caseless
+	}
+	if p.Flags.DotAll {
+		hp.Flags |= hyperscan.DotAll
+	}
+	if p.Flags.MultiLine {
+		hp.Flags |= hyperscan.MultiLine
+	}
+	if p.Flags.SingleMatch {
+		hp.Flags |= hyperscan.SingleMatch
+	}
+	if p.Flags.Utf8Mode {
+		hp.Flags |= hyperscan.Utf8Mode
+	}
+	if p.Flags.UnicodeProperty {
+		hp.Flags |= hyperscan.UnicodeProperty
+	}
+
+	if !hp.IsValid() {
+		return errors.New("can't validate the pattern")
+	}
+
+	return nil
+}
+
+func (p Pattern) Hash() string {
+	hash := sha256.New()
+	hash.Write([]byte(fmt.Sprintf("%s|%v|%v|%v", p.Regex, p.Flags, p.MinOccurrences, p.MaxOccurrences)))
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}
+
+func test() {
+	user := &Pattern{Regex: "Frank"}
+	b, err := json.Marshal(user)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(string(b))
+
+	p, _ := hyperscan.ParsePattern("/a/")
+	p1, _ := hyperscan.ParsePattern("/a/")
+	fmt.Println(p1.String(), p1.Flags)
+	//p1.Id = 1
+
+	fmt.Println(*p == *p1)
+	db, _ := hyperscan.NewBlockDatabase(p, p1)
+	s, _ := hyperscan.NewScratch(db)
+	db.Scan([]byte("Ciao"), s, onMatch, nil)
+
+
+
+
+}
+
+func onMatch(id uint, from uint64, to uint64, flags uint, context interface{}) error {
+	fmt.Println(id)
+
+	return nil
+}
\ No newline at end of file
diff --git a/travis_tests.sh b/scripts/travis_tests.sh
index 5c51823..5c51823 100755
--- a/travis_tests.sh
+++ b/scripts/travis_tests.sh
diff --git a/storage.go b/storage.go
index ea24780..3be56d7 100644
--- a/storage.go
+++ b/storage.go
@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"time"
@@ -11,13 +13,22 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
+const Connections = "connections"
+const ImportedPcaps = "imported_pcaps"
+const Rules = "rules"
+
+var NoFilters = UnorderedDocument{}
+
 const defaultConnectionTimeout = 10*time.Second
 const defaultOperationTimeout = 3*time.Second
 
 type Storage interface {
 	InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error)
+	InsertMany(ctx context.Context, collectionName string, documents []interface{}) ([]interface{}, error)
 	UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error)
+	UpdateMany(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error)
 	FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error)
+	Find(ctx context.Context, collectionName string, filter interface{}, results interface{}) error
 }
 
 type MongoStorage struct {
@@ -28,6 +39,14 @@ type MongoStorage struct {
 type OrderedDocument = bson.D
 type UnorderedDocument = bson.M
 
+func UniqueKey(timestamp time.Time, payload uint32) string {
+	var key [8]byte
+	binary.BigEndian.PutUint32(key[0:4], uint32(timestamp.Unix()))
+	binary.BigEndian.PutUint32(key[4:8], payload)
+
+	return hex.EncodeToString(key[:])
+}
+
 func NewMongoStorage(uri string, port int, database string) *MongoStorage {
 	opt := options.Client()
 	opt.ApplyURI(fmt.Sprintf("mongodb://%s:%v", uri, port))
@@ -38,8 +57,9 @@ func NewMongoStorage(uri string, port int, database string) *MongoStorage {
 
 	db := client.Database(database)
 	colls := map[string]*mongo.Collection{
-		"imported_pcaps": db.Collection("imported_pcaps"),
-		"connections": db.Collection("connections"),
+		Connections: db.Collection(Connections),
+		ImportedPcaps: db.Collection(ImportedPcaps),
+		Rules: db.Collection(Rules),
 	}
 
 	return &MongoStorage{
@@ -76,6 +96,26 @@ func (storage *MongoStorage) InsertOne(ctx context.Context, collectionName strin
 	return result.InsertedID, nil
 }
 
+func (storage *MongoStorage) InsertMany(ctx context.Context, collectionName string,
+	documents []interface{}) ([]interface{}, error) {
+
+	collection, ok := storage.collections[collectionName]
+	if !ok {
+		return nil, errors.New("invalid collection: " + collectionName)
+	}
+
+	if ctx == nil {
+		ctx, _ = context.WithTimeout(context.Background(), defaultOperationTimeout)
+	}
+
+	result, err := collection.InsertMany(ctx, documents)
+	if err != nil {
+		return nil, err
+	}
+
+	return result.InsertedIDs, nil
+}
+
 func (storage *MongoStorage) UpdateOne(ctx context.Context, collectionName string,
 	filter interface{}, update interface {}, upsert bool) (interface{}, error) {
 
@@ -150,3 +190,55 @@ func (storage *MongoStorage) FindOne(ctx context.Context, collectionName string,
 
 	return result, nil
 }
+
+type FindOperation struct {
+	options options.FindOptions
+}
+
+
+
+func (storage *MongoStorage) Find(ctx context.Context, collectionName string,
+	filter interface{}, results interface{}) error {
+
+	collection, ok := storage.collections[collectionName]
+	if !ok {
+		return errors.New("invalid collection: " + collectionName)
+	}
+
+	if ctx == nil {
+		ctx, _ = context.WithTimeout(context.Background(), defaultOperationTimeout)
+	}
+
+	options.FindOptions{
+		AllowDiskUse:        nil,
+		AllowPartialResults: nil,
+		BatchSize:           nil,
+		Collation:           nil,
+		Comment:             nil,
+		CursorType:          nil,
+		Hint:                nil,
+		Limit:               nil,
+		Max:                 nil,
+		MaxAwaitTime:        nil,
+		MaxTime:             nil,
+		Min:                 nil,
+		NoCursorTimeout:     nil,
+		OplogReplay:         nil,
+		Projection:          nil,
+		ReturnKey:           nil,
+		ShowRecordID:        nil,
+		Skip:                nil,
+		Snapshot:            nil,
+		Sort:                nil,
+	}
+	cursor, err := collection.Find(ctx, filter)
+	if err != nil {
+		return err
+	}
+	err = cursor.All(ctx, results)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/storage_test.go b/storage_test.go
index b46b60a..40440a4 100644
--- a/storage_test.go
+++ b/storage_test.go
@@ -1,10 +1,27 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"testing"
+	"time"
 )
 
-const testCollection = "characters"
+type a struct {
+	Id primitive.ObjectID `bson:"_id,omitempty"`
+	A string `bson:"a,omitempty"`
+	B int `bson:"b,omitempty"`
+	C time.Time `bson:"c,omitempty"`
+	D map[string]b `bson:"d"`
+	E []b `bson:"e,omitempty"`
+}
+
+type b struct {
+	A string `bson:"a,omitempty"`
+	B int `bson:"b,omitempty"`
+}
 
 func testInsert(t *testing.T) {
 	// insert a document in an invalid connection
@@ -88,3 +105,115 @@ func TestBasicOperations(t *testing.T) {
 	t.Run("testInsert", testInsert)
 	t.Run("testFindOne", testFindOne)
 }
+
+func TestInsertManyFindDocuments(t *testing.T) {
+	testTime := time.Now()
+	oid1, err := primitive.ObjectIDFromHex("ffffffffffffffffffffffff")
+	assert.Nil(t, err)
+
+	docs := []interface{}{
+		a{
+			A: "test0",
+			B: 0,
+			C: testTime,
+			D: map[string]b{
+				"first": {A: "0", B: 0},
+				"second": {A: "1", B: 1},
+			},
+			E: []b{
+				{A: "0", B: 0}, {A: "1", B: 0},
+			},
+		},
+		a{
+			Id: oid1,
+			A: "test1",
+			B: 1,
+			C: testTime,
+			D: map[string]b{},
+			E: []b{},
+		},
+		a{},
+	}
+
+	ids, err := storage.InsertMany(testContext, testInsertManyFindCollection, docs)
+	assert.Nil(t, err)
+	assert.Len(t, ids, 3)
+	assert.Equal(t, ids[1], oid1)
+
+	var results []a
+	err = storage.Find(testContext, testInsertManyFindCollection, NoFilters, &results)
+	assert.Nil(t, err)
+	assert.Len(t, results, 3)
+	doc0, doc1, doc2 := docs[0].(a), docs[1].(a), docs[2].(a)
+	assert.Equal(t, ids[0], results[0].Id)
+	assert.Equal(t, doc1.Id, results[1].Id)
+	assert.Equal(t, ids[2], results[2].Id)
+	assert.Equal(t, doc0.A, results[0].A)
+	assert.Equal(t, doc1.A, results[1].A)
+	assert.Equal(t, doc2.A, results[2].A)
+	assert.Equal(t, doc0.B, results[0].B)
+	assert.Equal(t, doc1.B, results[1].B)
+	assert.Equal(t, doc2.B, results[2].B)
+	assert.Equal(t, doc0.C.Unix(), results[0].C.Unix())
+	assert.Equal(t, doc1.C.Unix(), results[1].C.Unix())
+	assert.Equal(t, doc2.C.Unix(), results[2].C.Unix())
+	assert.Equal(t, doc0.D, results[0].D)
+	assert.Equal(t, doc1.D, results[1].D)
+	assert.Equal(t, doc2.D, results[2].D)
+	assert.Equal(t, doc0.E, results[0].E)
+	assert.Nil(t, results[1].E)
+	assert.Nil(t, results[2].E)
+}
+
+type testStorage struct {
+	insertFunc func(ctx context.Context, collectionName string, document interface{}) (interface{}, error)
+	insertManyFunc func(ctx context.Context, collectionName string, document []interface{}) ([]interface{}, error)
+	updateOne func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error)
+	updateMany func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error)
+	findOne func(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error)
+	find func(ctx context.Context, collectionName string, filter interface{}, results interface{}) error
+}
+
+func (ts testStorage) InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error) {
+	if ts.insertFunc != nil {
+		return ts.insertFunc(ctx, collectionName, document)
+	}
+	return nil, errors.New("not implemented")
+}
+
+func (ts testStorage) InsertMany(ctx context.Context, collectionName string, document []interface{}) ([]interface{}, error) {
+	if ts.insertFunc != nil {
+		return ts.insertManyFunc(ctx, collectionName, document)
+	}
+	return nil, errors.New("not implemented")
+}
+
+func (ts testStorage) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {},
+	upsert bool) (interface{}, error) {
+	if ts.updateOne != nil {
+		return ts.updateOne(ctx, collectionName, filter, update, upsert)
+	}
+	return nil, errors.New("not implemented")
+}
+
+func (ts testStorage) UpdateMany(ctx context.Context, collectionName string, filter interface{}, update interface {},
+	upsert bool) (interface{}, error) {
+	if ts.updateOne != nil {
+		return ts.updateMany(ctx, collectionName, filter, update, upsert)
+	}
+	return nil, errors.New("not implemented")
+}
+
+func (ts testStorage) FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) {
+	if ts.findOne != nil {
+		return ts.findOne(ctx, collectionName, filter)
+	}
+	return nil, errors.New("not implemented")
+}
+
+func (ts testStorage) Find(ctx context.Context, collectionName string, filter interface{}, results interface{}) error {
+	if ts.find != nil {
+		return ts.find(ctx, collectionName, filter, results)
+	}
+	return errors.New("not implemented")
+}
diff --git a/stream_handler_test.go b/stream_handler_test.go
index 425c1b7..cb5ecc7 100644
--- a/stream_handler_test.go
+++ b/stream_handler_test.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"github.com/flier/gohs/hyperscan"
 	"github.com/google/gopacket/layers"
@@ -44,12 +43,12 @@ func TestReassemblingEmptyStream(t *testing.T) {
 	assert.Zero(t, streamHandler.streamLength)
 	assert.Len(t, streamHandler.patternMatches, 0)
 
-	expected := 0
+	completed := false
 	streamHandler.connection.(*testConnectionHandler).onComplete = func(handler *StreamHandler) {
-		expected = 42
+		completed = true
 	}
 	streamHandler.ReassemblyComplete()
-	assert.Equal(t, 42, expected)
+	assert.Equal(t, true, completed)
 
 	err = scratch.Free()
 	require.Nil(t, err, "free scratch")
@@ -340,31 +339,3 @@ func (tch *testConnectionHandler) Patterns() hyperscan.StreamDatabase {
 func (tch *testConnectionHandler) Complete(handler *StreamHandler) {
 	tch.onComplete(handler)
 }
-
-type testStorage struct {
-	insertFunc func(ctx context.Context, collectionName string, document interface{}) (interface{}, error)
-	updateOne func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error)
-	findOne func(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error)
-}
-
-func (ts testStorage) InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error) {
-	if ts.insertFunc != nil {
-		return ts.insertFunc(ctx, collectionName, document)
-	}
-	return nil, errors.New("not implemented")
-}
-
-func (ts testStorage) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {},
-	upsert bool) (interface{}, error) {
-	if ts.updateOne != nil {
-		return ts.updateOne(ctx, collectionName, filter, update, upsert)
-	}
-	return nil, errors.New("not implemented")
-}
-
-func (ts testStorage) FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) {
-	if ts.insertFunc != nil {
-		return ts.findOne(ctx, collectionName, filter)
-	}
-	return nil, errors.New("not implemented")
-}