#!/usr/bin/perl
#
# 6/22/2004 Author: Joe Barbish, I bequeath this perl script to public domain.
# It can be copied and distributed for free by anyone to anyone by any manner.
#
# Description: This perl script is an official DShield client who's
# purpose is to read your FreeBSD ipfilter firewall ipmon log file and 
# convert the log records to the standard DShield reporting record format, 
# and imbed the converted records into the body of an email that gets
# sent to DShield for automatic addition to their database and abuse reporting
# to the offenders ISP if you are an subscribed DShield member.
# Script contains user customable defaults which can be overridden with 
# command line flags. 
#
# Last verification test June 22 2004 using FreeBSD Release 4.9 & 4.10 
#
##############################################################################
#
# DShield report format is a tab delimited format containing the following
# items (in this order):
#    1.date (YYYY-MM-DD HH:MM:SS tzHH:tzMM). tz means timezone from GMT
#    2.Dshield user # (0 if not subscribed DShield member)
#    3.count (number, used to summarize identical records, default=1)
#    4.source IP address (in 0.0.0.0 format)
#    5.source port
#    6.target IP address (in 0.0.0.0 format)
#    7.target port
#    8.protocol (either number or text like 'TCP','UDP'...)
#    9.TCP flags (S-SYN, A-ACK, F-FIN, U-URG, R-RST, P-PSH)
##############################################################################

# Note1 Script must have execute permission  chmod 760 dshield-freebsd.pl

# Note2 Be careful if you edit ipfilter log file. This script expects all 
#       log records to end with one blank and no blank records in log file.
#       Marker offset file will get hosed if log file ends with blank record
#       and not be able to position correctly on next run. Check log file
#       if you have positioning problems. When you look at marker file contents
#       first record is the number of positions indexed into file as starting
#       position of last log record processed. The second record is an image
#       copy of the last log record processed.
#
# Note3 The perl Net::Netmask function used below is not installed in the 
#       perl version that is delivered as part of the standard FreeBSD 
#       sysinstall process.
#
#       pkg_add -rv p5-Net-Netmask   will install it.  
#
#       (note capital letter N in package name so you spell it correctly)
#       After pkg_add completes, follow with rehash command to enable it.
#
#       There is also an cvsup port version of p5-Net-Netmask or you can use
#       the perl cpan method to add the module.

use Net::Netmask;
use POSIX (strftime);
use Getopt::Std;
use IO::File;

getopts("h:f:t:c:u:l:m:e:v:s:");
$verbose=$opt_v;

$version="DShield-FreeBSD-IPFilter v20040622";

if ($opt_h)
{
  print <<_EOF_;

* $version, Joe Barbish update v20040622

Usage: dshield-freebsd [ -h Display help info ] [ -f email_from ] 
                       [ -t email_to ] [ -c email_cc ] [ -u Dshield userid ]
                       [ -l /path/ipflogfile ] 
                       [ -m log_offset_file ] [ -e exclusion_list_file ]
                       [ -s sendmail_location ] [ -v1 turns on verbose debug ]
_EOF_
  exit
}

#
# User customable script default Parameter definitions
#

# DShield assigned user number (0 if not subscribed)
$userid=0;

# Sender email address (don't forget backslash before @)
# If subscribed to DShield this email address must
# match the email address you used to subscribe. 
#$email_from="root\@this.machine";

# The To email address to send your ipfilter converted log records
# to as content in the body of the email. (don't forget backslash before @)
# Testing send it to your self so you can verify things are working. 
#$email_to="root\@this.machine";

# live Production 
#$email_to="report\@dshield.org";

# The CC email address (don't forget backslash before @)
# Send copy of email to your self to verify and track regular submissions.
#$email_cc="root\@this.machine";
$email_cc="";

# Path and file name of your ipfilter log file
#$logfile="/var/log/ipf.log";
#$logfile="/var/log/security.0";


# Path and file name of your marker offset file.
# Used to know where last dshield-freebsd run stopped in the log.
# One will be created if one does not exist. 
#$marker="/root/bin/abuse.dshield.offset.marker";

# Exclusion list file (regex lines)
# Used to exclude ipfilter log records containing LAN IP address 
# or any other log records you want from
# being processed and sent to DShield. 
# DShield considers all reserved private Lan IP address ranges as invalid.
#
# If you have no exclusion file all the records in the ipfilter
# log file will be processed and emailed.
#
# DShield report exclusion list file command syntax.
#
# Format:
# Type=IP
#
# with type:
#   SI (Source IP)
#   TI (Target IP)
#   SP (Source Port)
#   TP (Target port)
# port=single value between 1-65535
# and IP:
#   216.240.32.0/24                     The preferred form.
#   216.240.32.0:255.255.255.0
#   216.240.32.0-255.255.255.0
#   216.240.32.0', '255.255.255.0
#   216.240.32.0', '0xffffff00
#   216.240.32.0 - 216.240.32.255
#   216.240.32.4                        A /32 block.
#   216.240.32                          Always a /24 block.
#   216.240                             Always a /16 block.
#   140                                 Always a /8 block.
#   216.240.32/24
#   216.240/16
#
# Example file content:
#SI=0.0.0.0
#TI=255.255.255.255
#SI=192.168.0.0/24
#TI=192.168.0.0/24
#SI=192.168.1.0/24
#TI=192.168.1.0/24
#SP=4661
#TP=4661
#TP=4662

# Path and file name of your exclusion file. 
#$exclusions="/etc/dshield.excluded.lst";
$exclusions="/root/bin/abuse.dshield.excluded.lst";

# Pattern to match ipmon lines.
# Change tun0 to your firewall external interface name
#$logpattern="ipmon.*tun0";
$logpattern="ipmon.*rl0";

# Default path to FreeBSD install delivered sendmail. 
# Option to change path in case you installed newer version.
$sendmail="/usr/sbin/sendmail";


##############################################################################
### From here, no changes are required #######################################
##############################################################################

# DShield requires all submitted firewall logs include the timezone so the
# records can be synchronized for an world wide view.
# This gets your system time zone which you set during sysinstall 
# or with tzsetup command.

($timezone = strftime("%z", localtime)) =~ s/(\d\d)(\d\d)/$1:$2/;


debug("Checking for flag override arguments\n");
# Command line flags to over ride script default arguments

if ($opt_f ne "")
{ $email_from=$opt_f; }

if ($opt_t ne "")
{ $email_to=$opt_t; }

if ($opt_c ne "")
{ $email_cc=$opt_c; }

if ($opt_u ne "")
{ $userid=$opt_u; }

if ($opt_l ne "")
{ $logfile=$opt_l; }

if ($opt_m ne "")
{ $marker=$opt_m; }

if ($opt_e ne "")
{ $exclusions=$opt_e; }

if ($opt_s ne "")
{ $sendmail=$opt_s; }

# Create run history file
system("date >> /var/log/abuse.dshield.run.history");


# Read marker offset file & save positioning info
debug("Loading offset file\n");
open(IN,$marker);
$offset_line=<IN>;  # offset number of text positions into file 
$offset_value=<IN>; # copy of last complete log line processed 
chop($offset_line);
chop($offset_value);
close(IN);
debug("offset_line = $offset_line\n");
debug("offset_value = $offset_value\n");

# Read excluded file & save info
debug("Loading exclusion file\n");
@excluded=();
open(IN,$exclusions);
while (<IN>)
{ 
 chop; s/^\s+//; 
 if ($_ eq "") { next; }
 if ($_ !~ /^#/) { push(@excluded,$_); }
}
close(IN);


debug("Opening log file\n");
$fd = new IO::File;
$fd->open($logfile,"r") || die "Cant open log file";

# position offset
if ($offset_line != 0) # not empty or absent offset file
{
  debug("Positioning offset ($offset_line)\n");
  $fd->seek($offset_line,0); # set to old pointer value
  $line=<$fd>;
  chop($line);
  debug("Comparing log record: $line\n");
  debug("To positioning record: $offset_value\n");

  if (($line ne $offset_value) && ($offset_value ne ""))
  # position is incorrect, line differs, set to 0
  {
    debug("Line differs, positioning to 0\n");
    $fd->seek(0,0);
  }
  else
  {
    debug("Line positioning matches\n");
  }
}

if ($fd->eof()) 
{ 
  $fd->close();
   close(OUT);
   print "This log file has been previously processed.\n";
   exit;
} 

# Prepare sendmail
debug("Piping to sendmail\n");
if ($verbose==1)
{ open(OUT,"| /bin/cat") || die "Cant open sendmail pipe"; }
else
{ open(OUT,"| $sendmail -t -oi") || die "Cant open sendmail pipe"; }
print(OUT "From: $email_from\n");
print(OUT "To: $email_to\n");
print(OUT "Cc: $email_cc\n");
print(OUT "Subject: FORMAT DSHIELD USERID $userid TZ $timezone ($version)\n\n");


$curpos=$fd->tell();
debug("File pointer located at $curpos, reading logs\n");

# init some control fields
$first_time = "YES";
$yes = "YES";
$prev_log_line_count=0;

# What year are we??
$year=1900+(localtime(time)) [5];


# Starting roll through log, building DShield record in email body
while (1)
{
  # Log line format is:
  # Aug 14 22:49:36 hostname ipmon[xx]: 22:49:35.901496 tun0 @0:16 b 200.60.255.45,4123 -> 80.13.151.54,135 PR tcp len 20 48 -S IN 

  $line_curpos=$fd->tell();
  $line=<$fd>;
  chop($line);
  debug("log record = $line\n");

  # parse line to extract relevant fields
  @f=split(/\s+/,$line);

  # re-init work fields to null  
  $log_line_date="";
  $log_line_time="";
  $log_line_ip_src="";
  $log_line_ip_trg="";
  $log_line_port_src="";
  $log_line_port_trg="";
  $log_line_proto="";
  $log_line_flags="";
  $log_line_author=$userid;
  $log_line_dupnum="";
  $log_line_dupsufx="";


  $month=convmonth($f[0]);
  $log_line_date=sprintf("%04d-%02d-%02d",$year,$month,$f[1]);

  $log_line_time=$f[2];


  # For ipmom log records that have dup counter field the parse line 
  # fields end up having different field displacement.
  $dupctr=$f[6];
  debug("Log line dup counter = $dupctr\n");
  $log_line_dupnum = substr($dupctr, 0, length($dupctr) -1);
  $log_line_dupsufx = substr($dupctr, -1, 1);
  debug("counter = $log_line_dupnum\n");
  debug("counter = $log_line_dupsufx\n");

  if ($log_line_dupsufx eq "x")
     {
       debug("this log rec has dup counter\n");
       @g=split(/,/,$f[10]);
       $log_line_ip_src=$g[0];
       $log_line_port_src=$g[1];

       @g=split(/,/,$f[12]);
       $log_line_ip_trg=$g[0];
       $log_line_port_trg=$g[1];
       $_=$f[14];
       tr/a-z/A-Z/;
       $log_line_proto=$_;
       if ($log_line_proto =~ /TCP/i)
       {
         $_=$f[18];
         s/-//;
         $log_line_flags=$_;
         if ($log_line_flags =~ /frag/i) { $log_line_flags=""; }
       }
     }
     else
     {
       debug("this log rec does not have dup counter\n");
       @g=split(/,/,$f[9]);
       $log_line_ip_src=$g[0];
       $log_line_port_src=$g[1];
       @g=split(/,/,$f[11]);
       $log_line_ip_trg=$g[0];
       $log_line_port_trg=$g[1];
       $_=$f[13];
       tr/a-z/A-Z/;
       $log_line_proto=$_;
      if ($log_line_proto =~ /TCP/i)
      {
        $_=$f[17];
        s/-//;
        $log_line_flags=$_;
        if ($log_line_flags =~ /frag/i) { $log_line_flags=""; }
      }
     }

  if ($first_time eq $yes)
  {
      debug("first time logic $first_time\n");
      # populate summarize logic control fields one time at start
      $prev_log_line_date=$log_line_date;
      $prev_log_line_time=$log_line_time;
      $prev_log_line_ip_src=$log_line_ip_src;
      $prev_log_line_port_src=$log_line_port_src;
      $prev_log_line_ip_trg=$log_line_ip_trg;
      $prev_log_line_port_trg=$log_line_port_trg;
      $prev_log_line_proto=$log_line_proto;
      $prev_log_line_flags=$log_line_flags;
      $first_time = "NO";
      debug("first time logic $first_time\n");
  }


  # logic to drop excluded log records based on exclusion file 
  if (($line =~ /$log_pattern/)
   && ($log_line_ip_src =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
   && ($log_line_ip_trg =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
   && (NotExcluded("SI",$log_line_ip_src))
   && (NotExcluded("SP",$log_line_port_src))
   && (NotExcluded("TI",$log_line_ip_trg))
   && (NotExcluded("TP",$log_line_port_trg)))
  {
    # logic to summarize dup log records in same seconds
    if (($prev_log_line_date eq $log_line_date)
     && ($prev_log_line_time eq $log_line_time)
     && ($prev_log_line_ip_src eq $log_line_ip_src)
     && ($prev_log_line_port_src eq $log_line_port_src)
     && ($prev_log_line_ip_trg eq $log_line_ip_trg)
     && ($prev_log_line_port_trg eq $log_line_port_trg)
     && ($prev_log_line_proto eq $log_line_proto)
     && ($prev_log_line_flags eq $log_line_flags))
        { $prev_log_line_count++; 
          debug("prev = current summarize logic $prev_log_line_count\n");}
    else
    {
      #load email body with dshield record format containing log contents
      debug("load email body line with log line contents\n");
      print(OUT "$prev_log_line_date $prev_log_line_time $timezone\t");
      print(OUT "$log_line_author\t$prev_log_line_count\t");
      print(OUT "$prev_log_line_ip_src\t$prev_log_line_port_src\t");
      print(OUT "$prev_log_line_ip_trg\t$prev_log_line_port_trg\t");
      print(OUT "$prev_log_line_proto\t$prev_log_line_flags\n");

      # populate summarize logic control fields
      $prev_log_line_date=$log_line_date;
      $prev_log_line_time=$log_line_time;
      $prev_log_line_ip_src=$log_line_ip_src;
      $prev_log_line_port_src=$log_line_port_src;
      $prev_log_line_ip_trg=$log_line_ip_trg;
      $prev_log_line_port_trg=$log_line_port_trg;
      $prev_log_line_proto=$log_line_proto;
      $prev_log_line_flags=$log_line_flags;

      $prev_log_line_count=1;
 
     if ($log_line_dupsufx eq "x") {$prev_log_line_count = $log_line_dupnum};
    }
  }
  
  if ($fd->eof()) 
  {
      debug("EOF load email body line with log line contents\n");
      print(OUT "$log_line_date $log_line_time $timezone\t");
      print(OUT "$log_line_author\t$prev_log_line_count\t");
      print(OUT "$log_line_ip_src\t$log_line_port_src\t");
      print(OUT "$log_line_ip_trg\t$log_line_port_trg\t");
      print(OUT "$log_line_proto\t$log_line_flags\n");
      last;
  }
}

$fd->close();
close(OUT);

# at this point all the ipfilter log records have been processed
# and the log file has been closed.
# Now write the positioning offset info to the marker file.
open(OUT,">$marker");
print(OUT "$line_curpos\n");
print(OUT "$line\n");
close(OUT);
exit;

############# every thing beyond this point are subroutines ############

sub convmonth()
{
  my %months_tab = (Jan=>1, Feb=>2, Mar=>3, Apr=>4, May=>5, Jun=>6, Jul=>7, Aug=>8, Sep=>9, Oct=>10, Nov=>11, Dec=>12);
              
  return $months_tab{$_[0]};
}

sub NotExcluded
{
  my ($excl_type)=@_[0];
  my ($tocheck)=@_[1];
  my ($i);
  my (@f);
  my ($t);

  for ($i=0;$i<@excluded;$i++)
  {
    @f=split(/=/,$excluded[$i]);
    $type=$f[0];
    $value=$f[1];

    if (($excl_type =~ /^SI$/i) || ($excl_type =~ /^TI$/))
    {
      $t=new Net::Netmask($value);
      if (($t->match($tocheck)) && ($excl_type eq $type)) { return(0); }
    }
    elsif (($excl_type =~ /^SP$/i) || ($excl_type =~ /^TP$/))
    {
      if (($value==$tocheck) && ($excl_type eq $type)) { return(0); }
    }
  }
  return(1);
}

sub debug
{
  if ($verbose==1)
  { print(STDERR @_); }
}