svn.andrew.net.au/scripts/report_iptables_ulog.py

#!/usr/bin/python

import optparse
import logging
import urllib
import sys

#
# See
# http://blog.steve.org.uk/wash_your_face_and_try_again__if_you_survive_.html
# http://blog.steve.org.uk/i_don_t_have_no_other_pants_.html
# for more background
# 
# Copyright (c) 2008 Andrew Pollock <me@andrew.net.au>
# 
# Permission to use, copy, distribute, modify, and otherwise knock yourself out
# granted under the terms of the GNU General Public Licence v2
#

__author__ = "Andrew Pollock <me@andrew.net.au>"
__license__ = "GPLv2"

HOSTNAME_FIELD=3
PREFIX_FIELD_START=4
FIELD_AFTER_PREFIX="IN="
SRC_FIELD_OFFSET=3


def setup_logging(options):
  """
  Initialise logging

  Args:
    options: options object
  """

  logging.getLogger("").setLevel(1)

  console = logging.StreamHandler()
  console_formatter = logging.Formatter("%(levelname)-8s %(message)s")
  console.setFormatter(console_formatter)
  if options.verbose:
    console.setLevel(logging.DEBUG)
  else:
    console.setLevel(logging.INFO)

  logging.getLogger("").addHandler(console)


def process_options():
  """
  Do all the command line option parsing

  Returns:
  an options object
  """

  USAGE="usage: %prog [options]"

  parser = optparse.OptionParser(usage=USAGE)
  parser.add_option(
    "-v", "--verbose",
    default=False,
    action="store_true",
    dest="verbose",
    help="increase verbosity",
  )
  parser.add_option(
    "-p", "--prefix",
    default="",
    action="store",
    dest="prefix",
    help="prefix to restrict matches to",
  )
  parser.add_option(
    "--host",
    default="",
    action="store",
    dest="hostname",
    help="host to restrict matches to",
  )
  parser.add_option(
    "-l",
    "--logfile",
    default="/var/log/ulog/syslogemu.log",
    action="store",
    dest="log",
    help="log file to process [default: %default]",
  )
  parser.add_option(
    "-n",
    "--dry-run",
    default=False,
    action="store_true",
    dest="dryrun",
    help="do nothing, just show what would happen",
  )
  (options, unused_args) = parser.parse_args()

  return options


def get_field_after_prefix(ulogentry):
  """
  Return the index of the field after the prefix

  Args:
    ulogentry: a list of strings representing one line of a ulog entry

  Returns:
    the index of the first field after the prefix
  """

  for component in xrange(PREFIX_FIELD_START, len(ulogentry)):
    if not ulogentry[component].startswith(FIELD_AFTER_PREFIX):
      continue
    else:
      break

  # If for some reason we reached the end of the line without finding
  # FIELD_AFTER_PREFIX, something is probably wrong

  if component == len(ulogentry):
    return 0
  else:
    return component


def process_log(options):
  """
  Process the log file

  Args:
    options object

  Returns:
    A set of IP addresses

  Discussion:

  A line of the log file can look like:
  Aug 31 06:46:06 caesar FORWARD too-hard-basket IN=eth0 OUT=eth1 MAC=00:08:02:52:
  49:d7:00:02:3b:01:f0:87:08:00  SRC=64.142.100.44 DST=172.16.0.7 LEN=128 TOS=00 P
  REC=0x00 TTL=59 ID=0 DF PROTO=ICMP TYPE=8 CODE=0 ID=42504 SEQ=6344

  Notably, the "FORWARD too-hard-basket" is a user definable string, which can
  contain whitespace, so therefore there's no predetermined position that the
  SRC= field can be found in, therefore for each line, we need to iterate over
  each whitespace-separated "word" until we find the right one. Bit of a bummer
  that.
  """

  ips = set()

  try:
    log = open(options.log)
  except IOError, e:
    logging.critical("Could not open %s (%s)" % (options.log, e.strerror))
    sys.exit(1)
  while True:
    log_entry = log.readline().split()
    if not log_entry:
      logging.debug("Reached the end of the file")
      break
    if len(log_entry) < PREFIX_FIELD_START + SRC_FIELD_OFFSET:
      # We've got a unexpected log entry
      # This should avoid any IndexError exceptions
      logging.critical("Ignoring unexpected log entry: %s" % (" ".join(log_entry)))
      continue
    if options.hostname and log_entry[HOSTNAME_FIELD] != options.hostname:
      logging.debug("Entry not for the host we're looking for")
      continue
    # We know where the prefix starts (if there is one)
    if options.prefix and log_entry[PREFIX_FIELD_START].startswith(FIELD_AFTER_PREFIX):
      # We have an entry with no prefix at all, but we're looking for entries
      # with a prefix so therefore this entry is automatically not what we're
      # looking for
      logging.debug("Entry has no prefix")
      continue
    else:
      # We have a prefixed entry, or we're looking for one
      next_field = get_field_after_prefix(log_entry)
      if next_field and \
        " ".join(log_entry[PREFIX_FIELD_START:next_field]) == options.prefix:
        # We have a valid line to work with
        ips.add(log_entry[next_field + SRC_FIELD_OFFSET].split("SRC=")[1])
      else:
        logging.debug("(2) Entry not for the prefix we're looking for: %s" 
          % " ".join(log_entry))
        continue
  log.close()

  return ips


def report_ips(options, ips):
  """
  Submit the IPs to the blacklist service

  Args:
    options: options object
    ips: set of IP addresses to report
  """

  # TODO(apollock): exception handling

  SUBMISSION_URL="http://blacklist.steve.org.uk/cgi-bin/report.cgi?src=%s"
  #SUBMISSION_URL="http://www.andrew.net.au/cgi-bin/report.cgi?src=%s"

  for ip in ips:
    if not options.dryrun:
      result = urllib.urlopen(SUBMISSION_URL % (ip))
      logging.info("%16s: %s" % (ip, "".join(result.readlines()).rstrip()))
      result.close()
    else:
      print SUBMISSION_URL % (ip)


def main():
  options = process_options()

  setup_logging(options)

  ips = process_log(options)

  if ips:
    report_ips(options, ips)


if __name__ == "__main__":
  main()
1	apollock	45	#!/usr/bin/python
2
3			import optparse
4			import logging
5			import urllib
6			import sys
7
8			#
9			# See
10			# http://blog.steve.org.uk/wash_your_face_and_try_again__if_you_survive_.html
11			# http://blog.steve.org.uk/i_don_t_have_no_other_pants_.html
12			# for more background
13			#
14			# Copyright (c) 2008 Andrew Pollock <me@andrew.net.au>
15			#
16			# Permission to use, copy, distribute, modify, and otherwise knock yourself out
17			# granted under the terms of the GNU General Public Licence v2
18			#
19
20			__author__ = "Andrew Pollock <me@andrew.net.au>"
21	apollock	46	__license__ = "GPLv2"
22	apollock	45
23			HOSTNAME_FIELD=3
24			PREFIX_FIELD_START=4
25			FIELD_AFTER_PREFIX="IN="
26			SRC_FIELD_OFFSET=3
27
28
29			def setup_logging(options):
30			"""
31			Initialise logging
32
33			Args:
34			options: options object
35			"""
36
37			logging.getLogger("").setLevel(1)
38
39			console = logging.StreamHandler()
40			console_formatter = logging.Formatter("%(levelname)-8s %(message)s")
41			console.setFormatter(console_formatter)
42			if options.verbose:
43			console.setLevel(logging.DEBUG)
44			else:
45			console.setLevel(logging.INFO)
46
47			logging.getLogger("").addHandler(console)
48
49
50			def process_options():
51			"""
52			Do all the command line option parsing
53
54			Returns:
55			an options object
56			"""
57
58			USAGE="usage: %prog [options]"
59
60			parser = optparse.OptionParser(usage=USAGE)
61			parser.add_option(
62			"-v", "--verbose",
63			default=False,
64			action="store_true",
65			dest="verbose",
66			help="increase verbosity",
67			)
68			parser.add_option(
69			"-p", "--prefix",
70			default="",
71			action="store",
72			dest="prefix",
73			help="prefix to restrict matches to",
74			)
75			parser.add_option(
76			"--host",
77			default="",
78			action="store",
79			dest="hostname",
80			help="host to restrict matches to",
81			)
82			parser.add_option(
83			"-l",
84			"--logfile",
85			default="/var/log/ulog/syslogemu.log",
86			action="store",
87			dest="log",
88			help="log file to process [default: %default]",
89			)
90			parser.add_option(
91			"-n",
92			"--dry-run",
93			default=False,
94			action="store_true",
95			dest="dryrun",
96			help="do nothing, just show what would happen",
97			)
98			(options, unused_args) = parser.parse_args()
99
100			return options
101
102
103	apollock	47	def get_field_after_prefix(ulogentry):
104	apollock	45	"""
105	apollock	47	Return the index of the field after the prefix
106	apollock	45
107			Args:
108			ulogentry: a list of strings representing one line of a ulog entry
109
110			Returns:
111	apollock	47	the index of the first field after the prefix
112	apollock	45	"""
113
114			for component in xrange(PREFIX_FIELD_START, len(ulogentry)):
115			if not ulogentry[component].startswith(FIELD_AFTER_PREFIX):
116	apollock	47	continue
117	apollock	45	else:
118			break
119
120			# If for some reason we reached the end of the line without finding
121			# FIELD_AFTER_PREFIX, something is probably wrong
122
123			if component == len(ulogentry):
124	apollock	47	return 0
125	apollock	45	else:
126	apollock	47	return component
127	apollock	45
128
129			def process_log(options):
130			"""
131			Process the log file
132
133			Args:
134			options object
135
136			Returns:
137			A set of IP addresses
138
139			Discussion:
140
141			A line of the log file can look like:
142			Aug 31 06:46:06 caesar FORWARD too-hard-basket IN=eth0 OUT=eth1 MAC=00:08:02:52:
143			49:d7:00:02:3b:01:f0:87:08:00 SRC=64.142.100.44 DST=172.16.0.7 LEN=128 TOS=00 P
144			REC=0x00 TTL=59 ID=0 DF PROTO=ICMP TYPE=8 CODE=0 ID=42504 SEQ=6344
145
146			Notably, the "FORWARD too-hard-basket" is a user definable string, which can
147			contain whitespace, so therefore there's no predetermined position that the
148			SRC= field can be found in, therefore for each line, we need to iterate over
149			each whitespace-separated "word" until we find the right one. Bit of a bummer
150			that.
151			"""
152
153			ips = set()
154
155			try:
156			log = open(options.log)
157			except IOError, e:
158			logging.critical("Could not open %s (%s)" % (options.log, e.strerror))
159			sys.exit(1)
160			while True:
161			log_entry = log.readline().split()
162			if not log_entry:
163			logging.debug("Reached the end of the file")
164			break
165	apollock	47	if len(log_entry) < PREFIX_FIELD_START + SRC_FIELD_OFFSET:
166			# We've got a unexpected log entry
167			# This should avoid any IndexError exceptions
168			logging.critical("Ignoring unexpected log entry: %s" % (" ".join(log_entry)))
169			continue
170	apollock	45	if options.hostname and log_entry[HOSTNAME_FIELD] != options.hostname:
171			logging.debug("Entry not for the host we're looking for")
172			continue
173			# We know where the prefix starts (if there is one)
174	apollock	47	if options.prefix and log_entry[PREFIX_FIELD_START].startswith(FIELD_AFTER_PREFIX):
175			# We have an entry with no prefix at all, but we're looking for entries
176			# with a prefix so therefore this entry is automatically not what we're
177			# looking for
178	apollock	45	logging.debug("Entry has no prefix")
179	apollock	47	continue
180	apollock	45	else:
181	apollock	47	# We have a prefixed entry, or we're looking for one
182			next_field = get_field_after_prefix(log_entry)
183			if next_field and \
184			" ".join(log_entry[PREFIX_FIELD_START:next_field]) == options.prefix:
185	apollock	45	# We have a valid line to work with
186			ips.add(log_entry[next_field + SRC_FIELD_OFFSET].split("SRC=")[1])
187			else:
188			logging.debug("(2) Entry not for the prefix we're looking for: %s"
189			% " ".join(log_entry))
190			continue
191			log.close()
192
193			return ips
194
195
196			def report_ips(options, ips):
197			"""
198			Submit the IPs to the blacklist service
199
200			Args:
201			options: options object
202			ips: set of IP addresses to report
203			"""
204
205	apollock	47	# TODO(apollock): exception handling
206
207	apollock	45	SUBMISSION_URL="http://blacklist.steve.org.uk/cgi-bin/report.cgi?src=%s"
208			#SUBMISSION_URL="http://www.andrew.net.au/cgi-bin/report.cgi?src=%s"
209
210			for ip in ips:
211			if not options.dryrun:
212			result = urllib.urlopen(SUBMISSION_URL % (ip))
213	apollock	47	logging.info("%16s: %s" % (ip, "".join(result.readlines()).rstrip()))
214	apollock	45	result.close()
215			else:
216			print SUBMISSION_URL % (ip)
217
218	apollock	46
219	apollock	45	def main():
220			options = process_options()
221
222			setup_logging(options)
223
224			ips = process_log(options)
225
226			if ips:
227			report_ips(options, ips)
228
229
230			if __name__ == "__main__":
231			main()