svn.andrew.net.au/scripts/ready2mirror.py

#!/usr/bin/python

import urllib2
import sys
import time
import stat
import os
from optparse import OptionParser

###
# Script to check if a local mirror is in sync with an HTTP-accessible remote
# mirror (based on the /debian/project/trace file)
#
# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
#
# Permission to use granted under the GNU General Public License v2

class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
        """The class you need to have to get useful IMS functionality"""

        def http_error_default(self, req, fp, code, msg, headers):
                """Returns the result code of the transaction"""

                result = urllib2.HTTPError(
                        req.get_full_url(), code, msg, headers, fp)
                result.status = code
                return result


def getLastModifiedTime(url):
        """Makes a request for url and returns the Last-Modified header"""

        request = urllib2.Request(url)
        opener = urllib2.build_opener()
        data = opener.open(request)
        return(data.headers.dict["last-modified"])


def URLisNewerThan(url, epoch):
        """Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""

        request = urllib2.Request(url)
        request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
        opener = urllib2.build_opener(DefaultErrorHandler())
        data = opener.open(request)
        if (hasattr(data, 'status') and data.status == 304):
                return False
        else:
                return True


def main():
        parser = OptionParser()

        parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
        parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
                help="Seconds to wait between checks for")
        parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
                help="Maximum number of interations before giving up")
        parser.add_option("-v", "--verbose", action="store_true", 
                dest="verbose", default=False,
                help="Print additional information about behaviour")

        (options, args) = parser.parse_args()

        if len(args) != 2:
                parser.print_usage()
        else:
        
                UPSTREAM=args[0]
                MIRROR_ROOT=args[1]
                upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
                local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)

                #upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
                upstream_timestamp = getLastModifiedTime(upstream_trace)

                if options.verbose:
                        print "Upstream trace file last modified on", upstream_timestamp

                local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]

                if options.verbose:
                        print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))

                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                        if options.verbose:
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)
                else:
                        if options.verbose:
                                print "I believe our mirror to be in sync with the upstream"
                        iterations = 0
                        while iterations < options.maxretries:
                                if options.verbose:
                                        print "Sleeping for %d seconds" % options.wait
                                time.sleep(options.wait)
                                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                                        upstream_timestamp = getLastModifiedTime(upstream_trace)
                                        break
                                else:
                                        if options.verbose:
                                                print "Upstream trace file last modified on", upstream_timestamp
                                        iterations += 1
                        else:
                                if options.verbose and iterations > 0:
                                        print "Giving up"
                                sys.exit(1)
                        if options.verbose:
                                print "Upstream trace file last modified on", upstream_timestamp
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)

if __name__ == '__main__':
        main()
1	apollock	1	#!/usr/bin/python
2
3			import urllib2
4			import sys
5			import time
6			import stat
7			import os
8			from optparse import OptionParser
9
10			###
11			# Script to check if a local mirror is in sync with an HTTP-accessible remote
12			# mirror (based on the /debian/project/trace file)
13			#
14			# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
15			#
16			# Permission to use granted under the GNU General Public License v2
17
18			class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
19			"""The class you need to have to get useful IMS functionality"""
20
21			def http_error_default(self, req, fp, code, msg, headers):
22			"""Returns the result code of the transaction"""
23
24			result = urllib2.HTTPError(
25			req.get_full_url(), code, msg, headers, fp)
26			result.status = code
27			return result
28
29
30			def getLastModifiedTime(url):
31			"""Makes a request for url and returns the Last-Modified header"""
32
33			request = urllib2.Request(url)
34			opener = urllib2.build_opener()
35			data = opener.open(request)
36			return(data.headers.dict["last-modified"])
37
38
39			def URLisNewerThan(url, epoch):
40			"""Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""
41
42			request = urllib2.Request(url)
43			request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
44			opener = urllib2.build_opener(DefaultErrorHandler())
45			data = opener.open(request)
46			if (hasattr(data, 'status') and data.status == 304):
47			return False
48			else:
49			return True
50
51
52			def main():
53			parser = OptionParser()
54
55			parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
56			parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
57			help="Seconds to wait between checks for")
58			parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
59			help="Maximum number of interations before giving up")
60			parser.add_option("-v", "--verbose", action="store_true",
61			dest="verbose", default=False,
62			help="Print additional information about behaviour")
63
64			(options, args) = parser.parse_args()
65
66			if len(args) != 2:
67			parser.print_usage()
68			else:
69
70			UPSTREAM=args[0]
71			MIRROR_ROOT=args[1]
72			upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
73			local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)
74
75			#upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
76			upstream_timestamp = getLastModifiedTime(upstream_trace)
77
78			if options.verbose:
79			print "Upstream trace file last modified on", upstream_timestamp
80
81			local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]
82
83			if options.verbose:
84			print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))
85
86			if URLisNewerThan(upstream_trace, local_trace_timestamp):
87			if options.verbose:
88			print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
89			sys.exit(0)
90			else:
91			if options.verbose:
92			print "I believe our mirror to be in sync with the upstream"
93			iterations = 0
94			while iterations < options.maxretries:
95			if options.verbose:
96			print "Sleeping for %d seconds" % options.wait
97			time.sleep(options.wait)
98			if URLisNewerThan(upstream_trace, local_trace_timestamp):
99			upstream_timestamp = getLastModifiedTime(upstream_trace)
100			break
101			else:
102			if options.verbose:
103			print "Upstream trace file last modified on", upstream_timestamp
104			iterations += 1
105			else:
106			if options.verbose and iterations > 0:
107			print "Giving up"
108			sys.exit(1)
109			if options.verbose:
110			print "Upstream trace file last modified on", upstream_timestamp
111			print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
112			sys.exit(0)
113
114			if __name__ == '__main__':
115			main()