diff options
| -rw-r--r-- | Dockerfile | 2 | ||||
| -rw-r--r-- | README.md | 3 | ||||
| -rw-r--r-- | caronte.go | 29 | ||||
| -rw-r--r-- | caronte_test.go | 15 | ||||
| -rw-r--r-- | docker-compose.testing.yml | 2 | ||||
| -rw-r--r-- | go.mod | 8 | ||||
| -rw-r--r-- | go.sum | 48 | ||||
| -rw-r--r-- | routes.go | 33 | ||||
| -rw-r--r-- | rules_manager.go | 242 | ||||
| -rwxr-xr-x | scripts/travis_tests.sh (renamed from travis_tests.sh) | 0 | ||||
| -rw-r--r-- | storage.go | 96 | ||||
| -rw-r--r-- | storage_test.go | 131 | ||||
| -rw-r--r-- | stream_handler_test.go | 35 |
13 files changed, 590 insertions, 54 deletions
@@ -1,4 +1,4 @@ -FROM eciavatta/caronte-env +FROM eciavatta/caronte-env:latest COPY . /caronte @@ -2,11 +2,10 @@ [](https://travis-ci.com/eciavatta/caronte) [](https://codecov.io/gh/eciavatta/caronte) -[![GPL License][license-shield]][license-url] +[](https://www.codacy.com/manual/eciavatta/caronte?utm_source=github.com&utm_medium=referral&utm_content=eciavatta/caronte&utm_campaign=Badge_Grade) <img align="left" src="https://divinacommedia.weebly.com/uploads/5/5/2/3/5523249/1299707879.jpg"> Caronte is a tool to analyze the network flow during capture the flag events of type attack/defence. It reassembles TCP packets captured in pcap files to rebuild TCP connections, and analyzes each connection to find user-defined patterns. The patterns can be defined as regex or using protocol specific rules. The connection flows are saved into a database and can be visualized with the web application. REST API are also provided. - @@ -1,26 +1,33 @@ package main import ( + "flag" "fmt" - "net" + "github.com/gin-gonic/gin" + "log" ) func main() { - // pattern.Flags |= hyperscan.SomLeftMost + // test(); return + mongoHost := flag.String("mongo-host", "localhost", "address of MongoDB") + mongoPort := flag.Int("mongo-port", 27017, "port of MongoDB") + dbName := flag.String("db-name", "caronte", "name of database to use") - storage := NewMongoStorage("localhost", 27017, "testing") + bindAddress := flag.String("bind-address", "0.0.0.0", "address where server is bind") + bindPort := flag.Int("bind-port", 3333, "port where server is bind") + + flag.Parse() + + storage := NewMongoStorage(*mongoHost, *mongoPort, *dbName) err := storage.Connect(nil) if err != nil { - panic(err) + log.Panicln("failed to connect to MongoDB:", err) } - importer := NewPcapImporter(storage, net.ParseIP("10.10.10.10")) - - sessionId, err := importer.ImportPcap("capture_00459_20190627165500.pcap") + router := gin.Default() + ApplicationRoutes(router) + err = router.Run(fmt.Sprintf("%s:%v", *bindAddress, *bindPort)) if err != nil { - fmt.Println(err) - } else { - fmt.Println(sessionId) + log.Panicln("failed to create the server:", err) } - } diff --git a/caronte_test.go b/caronte_test.go index cbf867d..9942086 100644 --- a/caronte_test.go +++ b/caronte_test.go @@ -6,6 +6,7 @@ import ( "fmt" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "log" "os" "testing" "time" @@ -14,6 +15,9 @@ import ( var storage Storage var testContext context.Context +const testInsertManyFindCollection = "testFi" +const testCollection = "characters" + func TestMain(m *testing.M) { mongoHost, ok := os.LookupEnv("MONGO_HOST") if !ok { @@ -31,15 +35,20 @@ func TestMain(m *testing.M) { panic("failed to create mongo client") } - db := client.Database(fmt.Sprintf("%x", uniqueDatabaseName[:31])) + dbName := fmt.Sprintf("%x", uniqueDatabaseName[:31]) + db := client.Database(dbName) + log.Println("using database", dbName) mongoStorage := MongoStorage{ client: client, - collections: map[string]*mongo.Collection{testCollection: db.Collection(testCollection)}, + collections: map[string]*mongo.Collection{ + testInsertManyFindCollection: db.Collection(testInsertManyFindCollection), + testCollection: db.Collection(testCollection), + }, } testContext, _ = context.WithTimeout(context.Background(), 10 * time.Second) - err = mongoStorage.Connect(nil) + err = mongoStorage.Connect(testContext) if err != nil { panic(err) } diff --git a/docker-compose.testing.yml b/docker-compose.testing.yml index 51997d6..ff33e3c 100644 --- a/docker-compose.testing.yml +++ b/docker-compose.testing.yml @@ -18,7 +18,7 @@ services: - mongo networks: - acheronet - command: "./travis_tests.sh" + command: "./scripts/travis_tests.sh" environment: MONGO_HOST: mongo MONGO_PORT: 27017 @@ -4,8 +4,14 @@ go 1.14 require ( github.com/flier/gohs v1.0.0 + github.com/gin-gonic/gin v1.6.2 + github.com/go-playground/validator/v10 v10.2.0 + github.com/golang/protobuf v1.3.5 // indirect github.com/google/gopacket v1.1.17 - github.com/stretchr/testify v1.3.0 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/stretchr/testify v1.4.0 go.mongodb.org/mongo-driver v1.3.1 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 + golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect ) @@ -4,6 +4,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/flier/gohs v1.0.0 h1:Q0mmufGWTigzKb140WmJ0+k3EGAf335Qgv/pz5SOPvU= github.com/flier/gohs v1.0.0/go.mod h1:Jlg6A1xXSMhPorF74/LkYHkCHZ87Txi8CqIHHyIKgKg= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM= +github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= +github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -30,10 +43,15 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.14/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= @@ -45,6 +63,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= @@ -57,8 +77,20 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -83,8 +115,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= @@ -112,6 +150,10 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2 h1:T5DasATyLQfmbTpfEXx/IOL9vfjzW6up+ZDkmHvIf2s= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0= +golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -124,4 +166,10 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/routes.go b/routes.go new file mode 100644 index 0000000..f44cff7 --- /dev/null +++ b/routes.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/go-playground/validator/v10" + "log" + "net/http" +) + +func ApplicationRoutes(engine *gin.Engine) { + engine.Static("/", "./frontend/build") + + api := engine.Group("/api") + { + api.POST("/rules", func(c *gin.Context) { + var rule Rule + + if err := c.ShouldBindJSON(&rule); err != nil { + for _, fieldErr := range err.(validator.ValidationErrors) { + log.Println(fieldErr) + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("field '%v' does not respect the %v(%v) rule", + fieldErr.Field(), fieldErr.Tag(), fieldErr.Param()), + }) + return // exit on first error + } + } + + c.JSON(200, rule) + }) + } +} diff --git a/rules_manager.go b/rules_manager.go new file mode 100644 index 0000000..d6e9aaa --- /dev/null +++ b/rules_manager.go @@ -0,0 +1,242 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "github.com/flier/gohs/hyperscan" + "go.mongodb.org/mongo-driver/bson/primitive" + "log" + "sync" + "time" +) + + +type RegexFlags struct { + Caseless bool `json:"caseless"` // Set case-insensitive matching. + DotAll bool `json:"dot_all"` // Matching a `.` will not exclude newlines. + MultiLine bool `json:"multi_line"` // Set multi-line anchoring. + SingleMatch bool `json:"single_match"` // Set single-match only mode. + Utf8Mode bool `json:"utf_8_mode"` // Enable UTF-8 mode for this expression. + UnicodeProperty bool `json:"unicode_property"` // Enable Unicode property support for this expression +} + +type Pattern struct { + Regex string `json:"regex"` + Flags RegexFlags `json:"flags"` + MinOccurrences int `json:"min_occurrences"` + MaxOccurrences int `json:"max_occurrences"` + internalId int + compiledPattern *hyperscan.Pattern +} + +type Filter struct { + ServicePort int + ClientAddress string + ClientPort int + MinDuration int + MaxDuration int + MinPackets int + MaxPackets int + MinSize int + MaxSize int +} + +type Rule struct { + Id string `json:"-" bson:"_id,omitempty"` + Name string `json:"name" binding:"required,min=3" bson:"name"` + Color string `json:"color" binding:"required,hexcolor" bson:"color"` + Notes string `json:"notes" bson:"notes,omitempty"` + Enabled bool `json:"enabled" bson:"enabled"` + Patterns []Pattern `json:"patterns" binding:"required,min=1" bson:"patterns"` + Filter Filter `json:"filter" bson:"filter,omitempty"` + Version int64 `json:"version" bson:"version"` +} + +type RulesManager struct { + storage Storage + rules map[string]Rule + rulesByName map[string]Rule + ruleIndex int + patterns map[string]Pattern + mPatterns sync.Mutex + databaseUpdated chan interface{} +} + +func NewRulesManager(storage Storage) RulesManager { + return RulesManager{ + storage: storage, + rules: make(map[string]Rule), + patterns: make(map[string]Pattern), + mPatterns: sync.Mutex{}, + } +} + + + +func (rm RulesManager) LoadRules() error { + var rules []Rule + if err := rm.storage.Find(nil, Rules, NoFilters, &rules); err != nil { + return err + } + + var version int64 + for _, rule := range rules { + if err := rm.validateAndAddRuleLocal(&rule); err != nil { + log.Printf("failed to import rule %s: %s\n", rule.Name, err) + continue + } + if rule.Version > version { + version = rule.Version + } + } + + rm.ruleIndex = len(rules) + return rm.generateDatabase(0) +} + +func (rm RulesManager) AddRule(context context.Context, rule Rule) (string, error) { + rm.mPatterns.Lock() + + rule.Id = UniqueKey(time.Now(), uint32(rm.ruleIndex)) + rule.Enabled = true + + if err := rm.validateAndAddRuleLocal(&rule); err != nil { + rm.mPatterns.Unlock() + return "", err + } + rm.mPatterns.Unlock() + + if _, err := rm.storage.InsertOne(context, Rules, rule); err != nil { + return "", err + } + + return rule.Id, rm.generateDatabase(rule.Id) +} + + + +func (rm RulesManager) validateAndAddRuleLocal(rule *Rule) error { + if _, alreadyPresent := rm.rulesByName[rule.Name]; alreadyPresent { + return errors.New("rule name must be unique") + } + + newPatterns := make(map[string]Pattern) + for i, pattern := range rule.Patterns { + hash := pattern.Hash() + if existingPattern, isPresent := rm.patterns[hash]; isPresent { + rule.Patterns[i] = existingPattern + continue + } + err := pattern.BuildPattern() + if err != nil { + return err + } + pattern.internalId = len(rm.patterns) + len(newPatterns) + newPatterns[hash] = pattern + } + + for key, value := range newPatterns { + rm.patterns[key] = value + } + + rm.rules[rule.Id] = *rule + rm.rulesByName[rule.Name] = *rule + + return nil +} + +func (rm RulesManager) generateDatabase(version string) error { + patterns := make([]*hyperscan.Pattern, len(rm.patterns)) + for _, pattern := range rm.patterns { + patterns = append(patterns, pattern.compiledPattern) + } + database, err := hyperscan.NewStreamDatabase(patterns...) + if err != nil { + return err + } + + rm.databaseUpdated <- database + return nil +} + + +func (p Pattern) BuildPattern() error { + if p.compiledPattern != nil { + return nil + } + if p.MinOccurrences <= 0 { + return errors.New("min_occurrences can't be lower than zero") + } + if p.MaxOccurrences != -1 && p.MinOccurrences < p.MinOccurrences { + return errors.New("max_occurrences can't be lower than min_occurrences") + } + + hp, err := hyperscan.ParsePattern(fmt.Sprintf("/%s/", p.Regex)) + if err != nil { + return err + } + + if p.Flags.Caseless { + hp.Flags |= hyperscan.Caseless + } + if p.Flags.DotAll { + hp.Flags |= hyperscan.DotAll + } + if p.Flags.MultiLine { + hp.Flags |= hyperscan.MultiLine + } + if p.Flags.SingleMatch { + hp.Flags |= hyperscan.SingleMatch + } + if p.Flags.Utf8Mode { + hp.Flags |= hyperscan.Utf8Mode + } + if p.Flags.UnicodeProperty { + hp.Flags |= hyperscan.UnicodeProperty + } + + if !hp.IsValid() { + return errors.New("can't validate the pattern") + } + + return nil +} + +func (p Pattern) Hash() string { + hash := sha256.New() + hash.Write([]byte(fmt.Sprintf("%s|%v|%v|%v", p.Regex, p.Flags, p.MinOccurrences, p.MaxOccurrences))) + return fmt.Sprintf("%x", hash.Sum(nil)) +} + +func test() { + user := &Pattern{Regex: "Frank"} + b, err := json.Marshal(user) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(string(b)) + + p, _ := hyperscan.ParsePattern("/a/") + p1, _ := hyperscan.ParsePattern("/a/") + fmt.Println(p1.String(), p1.Flags) + //p1.Id = 1 + + fmt.Println(*p == *p1) + db, _ := hyperscan.NewBlockDatabase(p, p1) + s, _ := hyperscan.NewScratch(db) + db.Scan([]byte("Ciao"), s, onMatch, nil) + + + + +} + +func onMatch(id uint, from uint64, to uint64, flags uint, context interface{}) error { + fmt.Println(id) + + return nil +}
\ No newline at end of file diff --git a/travis_tests.sh b/scripts/travis_tests.sh index 5c51823..5c51823 100755 --- a/travis_tests.sh +++ b/scripts/travis_tests.sh @@ -2,6 +2,8 @@ package main import ( "context" + "encoding/binary" + "encoding/hex" "errors" "fmt" "time" @@ -11,13 +13,22 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) +const Connections = "connections" +const ImportedPcaps = "imported_pcaps" +const Rules = "rules" + +var NoFilters = UnorderedDocument{} + const defaultConnectionTimeout = 10*time.Second const defaultOperationTimeout = 3*time.Second type Storage interface { InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error) + InsertMany(ctx context.Context, collectionName string, documents []interface{}) ([]interface{}, error) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) + UpdateMany(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) + Find(ctx context.Context, collectionName string, filter interface{}, results interface{}) error } type MongoStorage struct { @@ -28,6 +39,14 @@ type MongoStorage struct { type OrderedDocument = bson.D type UnorderedDocument = bson.M +func UniqueKey(timestamp time.Time, payload uint32) string { + var key [8]byte + binary.BigEndian.PutUint32(key[0:4], uint32(timestamp.Unix())) + binary.BigEndian.PutUint32(key[4:8], payload) + + return hex.EncodeToString(key[:]) +} + func NewMongoStorage(uri string, port int, database string) *MongoStorage { opt := options.Client() opt.ApplyURI(fmt.Sprintf("mongodb://%s:%v", uri, port)) @@ -38,8 +57,9 @@ func NewMongoStorage(uri string, port int, database string) *MongoStorage { db := client.Database(database) colls := map[string]*mongo.Collection{ - "imported_pcaps": db.Collection("imported_pcaps"), - "connections": db.Collection("connections"), + Connections: db.Collection(Connections), + ImportedPcaps: db.Collection(ImportedPcaps), + Rules: db.Collection(Rules), } return &MongoStorage{ @@ -76,6 +96,26 @@ func (storage *MongoStorage) InsertOne(ctx context.Context, collectionName strin return result.InsertedID, nil } +func (storage *MongoStorage) InsertMany(ctx context.Context, collectionName string, + documents []interface{}) ([]interface{}, error) { + + collection, ok := storage.collections[collectionName] + if !ok { + return nil, errors.New("invalid collection: " + collectionName) + } + + if ctx == nil { + ctx, _ = context.WithTimeout(context.Background(), defaultOperationTimeout) + } + + result, err := collection.InsertMany(ctx, documents) + if err != nil { + return nil, err + } + + return result.InsertedIDs, nil +} + func (storage *MongoStorage) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) { @@ -150,3 +190,55 @@ func (storage *MongoStorage) FindOne(ctx context.Context, collectionName string, return result, nil } + +type FindOperation struct { + options options.FindOptions +} + + + +func (storage *MongoStorage) Find(ctx context.Context, collectionName string, + filter interface{}, results interface{}) error { + + collection, ok := storage.collections[collectionName] + if !ok { + return errors.New("invalid collection: " + collectionName) + } + + if ctx == nil { + ctx, _ = context.WithTimeout(context.Background(), defaultOperationTimeout) + } + + options.FindOptions{ + AllowDiskUse: nil, + AllowPartialResults: nil, + BatchSize: nil, + Collation: nil, + Comment: nil, + CursorType: nil, + Hint: nil, + Limit: nil, + Max: nil, + MaxAwaitTime: nil, + MaxTime: nil, + Min: nil, + NoCursorTimeout: nil, + OplogReplay: nil, + Projection: nil, + ReturnKey: nil, + ShowRecordID: nil, + Skip: nil, + Snapshot: nil, + Sort: nil, + } + cursor, err := collection.Find(ctx, filter) + if err != nil { + return err + } + err = cursor.All(ctx, results) + if err != nil { + return err + } + + return nil +} diff --git a/storage_test.go b/storage_test.go index b46b60a..40440a4 100644 --- a/storage_test.go +++ b/storage_test.go @@ -1,10 +1,27 @@ package main import ( + "context" + "errors" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson/primitive" "testing" + "time" ) -const testCollection = "characters" +type a struct { + Id primitive.ObjectID `bson:"_id,omitempty"` + A string `bson:"a,omitempty"` + B int `bson:"b,omitempty"` + C time.Time `bson:"c,omitempty"` + D map[string]b `bson:"d"` + E []b `bson:"e,omitempty"` +} + +type b struct { + A string `bson:"a,omitempty"` + B int `bson:"b,omitempty"` +} func testInsert(t *testing.T) { // insert a document in an invalid connection @@ -88,3 +105,115 @@ func TestBasicOperations(t *testing.T) { t.Run("testInsert", testInsert) t.Run("testFindOne", testFindOne) } + +func TestInsertManyFindDocuments(t *testing.T) { + testTime := time.Now() + oid1, err := primitive.ObjectIDFromHex("ffffffffffffffffffffffff") + assert.Nil(t, err) + + docs := []interface{}{ + a{ + A: "test0", + B: 0, + C: testTime, + D: map[string]b{ + "first": {A: "0", B: 0}, + "second": {A: "1", B: 1}, + }, + E: []b{ + {A: "0", B: 0}, {A: "1", B: 0}, + }, + }, + a{ + Id: oid1, + A: "test1", + B: 1, + C: testTime, + D: map[string]b{}, + E: []b{}, + }, + a{}, + } + + ids, err := storage.InsertMany(testContext, testInsertManyFindCollection, docs) + assert.Nil(t, err) + assert.Len(t, ids, 3) + assert.Equal(t, ids[1], oid1) + + var results []a + err = storage.Find(testContext, testInsertManyFindCollection, NoFilters, &results) + assert.Nil(t, err) + assert.Len(t, results, 3) + doc0, doc1, doc2 := docs[0].(a), docs[1].(a), docs[2].(a) + assert.Equal(t, ids[0], results[0].Id) + assert.Equal(t, doc1.Id, results[1].Id) + assert.Equal(t, ids[2], results[2].Id) + assert.Equal(t, doc0.A, results[0].A) + assert.Equal(t, doc1.A, results[1].A) + assert.Equal(t, doc2.A, results[2].A) + assert.Equal(t, doc0.B, results[0].B) + assert.Equal(t, doc1.B, results[1].B) + assert.Equal(t, doc2.B, results[2].B) + assert.Equal(t, doc0.C.Unix(), results[0].C.Unix()) + assert.Equal(t, doc1.C.Unix(), results[1].C.Unix()) + assert.Equal(t, doc2.C.Unix(), results[2].C.Unix()) + assert.Equal(t, doc0.D, results[0].D) + assert.Equal(t, doc1.D, results[1].D) + assert.Equal(t, doc2.D, results[2].D) + assert.Equal(t, doc0.E, results[0].E) + assert.Nil(t, results[1].E) + assert.Nil(t, results[2].E) +} + +type testStorage struct { + insertFunc func(ctx context.Context, collectionName string, document interface{}) (interface{}, error) + insertManyFunc func(ctx context.Context, collectionName string, document []interface{}) ([]interface{}, error) + updateOne func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) + updateMany func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) + findOne func(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) + find func(ctx context.Context, collectionName string, filter interface{}, results interface{}) error +} + +func (ts testStorage) InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error) { + if ts.insertFunc != nil { + return ts.insertFunc(ctx, collectionName, document) + } + return nil, errors.New("not implemented") +} + +func (ts testStorage) InsertMany(ctx context.Context, collectionName string, document []interface{}) ([]interface{}, error) { + if ts.insertFunc != nil { + return ts.insertManyFunc(ctx, collectionName, document) + } + return nil, errors.New("not implemented") +} + +func (ts testStorage) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {}, + upsert bool) (interface{}, error) { + if ts.updateOne != nil { + return ts.updateOne(ctx, collectionName, filter, update, upsert) + } + return nil, errors.New("not implemented") +} + +func (ts testStorage) UpdateMany(ctx context.Context, collectionName string, filter interface{}, update interface {}, + upsert bool) (interface{}, error) { + if ts.updateOne != nil { + return ts.updateMany(ctx, collectionName, filter, update, upsert) + } + return nil, errors.New("not implemented") +} + +func (ts testStorage) FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) { + if ts.findOne != nil { + return ts.findOne(ctx, collectionName, filter) + } + return nil, errors.New("not implemented") +} + +func (ts testStorage) Find(ctx context.Context, collectionName string, filter interface{}, results interface{}) error { + if ts.find != nil { + return ts.find(ctx, collectionName, filter, results) + } + return errors.New("not implemented") +} diff --git a/stream_handler_test.go b/stream_handler_test.go index 425c1b7..cb5ecc7 100644 --- a/stream_handler_test.go +++ b/stream_handler_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "github.com/flier/gohs/hyperscan" "github.com/google/gopacket/layers" @@ -44,12 +43,12 @@ func TestReassemblingEmptyStream(t *testing.T) { assert.Zero(t, streamHandler.streamLength) assert.Len(t, streamHandler.patternMatches, 0) - expected := 0 + completed := false streamHandler.connection.(*testConnectionHandler).onComplete = func(handler *StreamHandler) { - expected = 42 + completed = true } streamHandler.ReassemblyComplete() - assert.Equal(t, 42, expected) + assert.Equal(t, true, completed) err = scratch.Free() require.Nil(t, err, "free scratch") @@ -340,31 +339,3 @@ func (tch *testConnectionHandler) Patterns() hyperscan.StreamDatabase { func (tch *testConnectionHandler) Complete(handler *StreamHandler) { tch.onComplete(handler) } - -type testStorage struct { - insertFunc func(ctx context.Context, collectionName string, document interface{}) (interface{}, error) - updateOne func(ctx context.Context, collectionName string, filter interface{}, update interface {}, upsert bool) (interface{}, error) - findOne func(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) -} - -func (ts testStorage) InsertOne(ctx context.Context, collectionName string, document interface{}) (interface{}, error) { - if ts.insertFunc != nil { - return ts.insertFunc(ctx, collectionName, document) - } - return nil, errors.New("not implemented") -} - -func (ts testStorage) UpdateOne(ctx context.Context, collectionName string, filter interface{}, update interface {}, - upsert bool) (interface{}, error) { - if ts.updateOne != nil { - return ts.updateOne(ctx, collectionName, filter, update, upsert) - } - return nil, errors.New("not implemented") -} - -func (ts testStorage) FindOne(ctx context.Context, collectionName string, filter interface{}) (UnorderedDocument, error) { - if ts.insertFunc != nil { - return ts.findOne(ctx, collectionName, filter) - } - return nil, errors.New("not implemented") -} |