modsec-auditlog-collector.pl

#!/usr/bin/perl
#
# ModSecurity for Apache (http://www.modsecurity.org)
# Copyright (c) 2002-2006 Thinking Stone (http://www.thinkingstone.com)
#
# $Id: modsec-auditlog-collector.pl,v 1.1.2.3 2006/01/31 11:27:45 ivanr Exp $
#
# This is a proof-of-concept script that listens to the
# audit log in real time and submits the entries to
# a remote HTTP server. This code is not suitable for
# non-trivial production use since it can only submit
# one audit log entry at a time, plus it does not handle
# errors gracefully.
#
# Usage:
#
# 1) Enter the correct parameters $CONSOLE_* below
#
# 2) Configure ModSecurity to use this script for
#    concurrent audit logging index:
#
#    SecAuditEngine RelevantOnly
#    SecAuditLogType Concurrent
#    SecAuditLogParts ABCDEFGHZ
#    SecAuditLogStorageDir /path/to/auditlog/data/
#    SecAuditLog "|/path/to/modsec-auditlog-collector.pl \
#        /path/to/auditlog/data/ \
#        /path/to/auditlog/index"
#
# 3) Restart Apache.

use MIME::Base64();
use IO::Socket::INET;

my $CONSOLE_URI = "/rpc/auditLogReceiver";
my $CONSOLE_HOST = "127.0.0.1";
my $CONSOLE_PORT = "8886";
my $CONSOLE_USERNAME = "test";
my $CONSOLE_PASSWORD = "sensor";

# ---------------------------------------------------

my $logline_regex = "";

# hostname
$logline_regex .= "^(\\S+)";
# remote host, remote username, local username
$logline_regex .= "\\ (\\S+)\\ (\\S+)\\ (\\S+)";
# date, time, and gmt offset
$logline_regex .= "\\ \\[([^:]+):(\\d+:\\d+:\\d+)\\ ([^\\]]+)\\]";
# request method + request uri + protocol (as one field)
$logline_regex .= "\\ \"(.*)\"";
# status, bytes out
$logline_regex .= "\\ (\\d+)\\ (\\S+)";
# referer, user_agent
$logline_regex .= "\\ \"(.*)\"\\ \"(.*)\"";
# uniqueid, sessionid
$logline_regex .= "\\ (\\S+)\\ \"(.*)\"";
# filename, offset, size
$logline_regex .= "\\ (\\S+)\\ (\\d+)\\ (\\d+)";
# hash
$logline_regex .= "\\ (\\S+)";
# the rest (always keep this part of the regex)
$logline_regex .= "(.*)\$";

my $therequest_regex = "(\\S+)\\ (.*?)\\ (\\S+)";

sub send_entry {
  my ($file_name, $file_offset, $file_size, $hash, $summary) = @_;
  my $buffer;

  if (!open(F, $file_name)) {
    print LOG "> Could not open file $file_name.\n";
    return;
  }

  binmode F;

  $socket = IO::Socket::INET->new(Proto => 'tcp', PeerAddr => $CONSOLE_HOST, PeerPort => $CONSOLE_PORT, Timeout => 10);
  binmode $socket;

  if (!$socket) {
    print LOG "> Failed to open socket.\n";
    return;
  }

  $socket->autoflush(1);

  my $credentials = MIME::Base64::encode($CONSOLE_USERNAME . ":" . $CONSOLE_PASSWORD);
  chomp($credentials);

  print $socket "PUT $CONSOLE_URI HTTP/1.0\r\n";
  print $socket "Content-Length: " . $file_size . "\r\n";
  print $socket "Authorization: Basic " . $credentials . "\r\n";
  print $socket "X-ForensicLog-Summary: " . $summary . "\r\n";
  print $socket "X-Content-Hash: " . $hash . "\r\n";
  print $socket "\r\n";

  # send file contents
  while (
    read(F, $buffer, 8192)
    and print $socket $buffer
  ) {};
  close(F);

  my $status = 0;
  while(<$socket>) {
    # print "> $_";
    if (($status == 0) && (/^HTTP\/[0-9]\.[0-9] ([0-9]+).+$/)) {
      $status = $1;
    }
  }

  print LOG "> Status: " . $status . "\n";
  close($socket);
}

# -- Main --------------------------------------------------------------------

if (@ARGV != 2) {
    print "Usage: modsec-auditlog-collector auditlog-folder auditlog-index\n";
    exit;
}

my($folder, $index) = @ARGV;

open(LOG, ">>$index") || die("Failed to open: $index\n");
$| = 1, select $_ for select LOG;


while(<STDIN>) {
    # print LOG "Line: $_";

    chomp();
    my $summary = $_;

    next if (/^$/);

    my @parsed_logline = /$logline_regex/x;
    if (@parsed_logline == 0) {
        print LOG "> Failed to parse line: " . $_ . "\n";
    } else {
      (
          $request{"hostname"},
          $request{"remote_ip"},
          $request{"remote_username"},
          $request{"username"},
          $request{"date"},
          $request{"time"},
          $request{"gmt_offset"},
          $request{"the_request"},
          $request{"status"},
          $request{"bytes_out"},
          $request{"referer"},
          $request{"user_agent"},
          $request{"unique_id"},
          $request{"session_id"},
          $request{"filename"},
          $request{"file_offset"},
          $request{"file_size"},
          $request{"hash"},
          $request{"the_rest"}
      ) = @parsed_logline;

      $_ = $request{"the_request"};
      my @parsed_therequest = /$therequest_regex/x;
      if (@parsed_therequest == 0) {
        $request{"invalid"} = "1";
        $request{"request_method"} = "";
        $request{"request_uri"} = "";
        $request{"protocol"} = "";
      } else {
        (
          $request{"request_method"},
          $request{"request_uri"},
          $request{"protocol"}
        ) = @parsed_therequest;
        }

      print LOG ($summary . "\n");
      send_entry($abs_file_name = $folder . "/" . $request{"filename"}, $request{"file_offset"}, $request{"file_size"}, $request{"hash"}, $summary);
    }
}

close(LOG);