svn.andrew.net.au/scripts/ready2mirror.py

#!/usr/bin/python

import urllib2
import sys
import time
import stat
import os
from optparse import OptionParser

###
# Script to check if a local mirror is in sync with an HTTP-accessible remote
# mirror (based on the /debian/project/trace file)
#
# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
#
# Permission to use granted under the GNU General Public License v2

class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
        """The class you need to have to get useful IMS functionality"""

        def http_error_default(self, req, fp, code, msg, headers):
                """Returns the result code of the transaction"""

                result = urllib2.HTTPError(
                        req.get_full_url(), code, msg, headers, fp)
                result.status = code
                return result


def getLastModifiedTime(url):
        """Makes a request for url and returns the Last-Modified header"""

        request = urllib2.Request(url)
        opener = urllib2.build_opener()
        data = opener.open(request)
        return(data.headers.dict["last-modified"])


def URLisNewerThan(url, epoch):
        """Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""

        request = urllib2.Request(url)
        request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
        opener = urllib2.build_opener(DefaultErrorHandler())
        data = opener.open(request)
        if (hasattr(data, 'status') and data.status == 304):
                return False
        else:
                return True


def main():
        parser = OptionParser()

        parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
        parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
                help="Seconds to wait between checks for")
        parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
                help="Maximum number of interations before giving up")
        parser.add_option("-v", "--verbose", action="store_true", 
                dest="verbose", default=False,
                help="Print additional information about behaviour")

        (options, args) = parser.parse_args()

        if len(args) != 2:
                parser.print_usage()
        else:
        
                UPSTREAM=args[0]
                MIRROR_ROOT=args[1]
                upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
                local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)

                #upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
                upstream_timestamp = getLastModifiedTime(upstream_trace)

                if options.verbose:
                        print "Upstream trace file last modified on", upstream_timestamp

                local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]

                if options.verbose:
                        print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))

                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                        if options.verbose:
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)
                else:
                        if options.verbose:
                                print "I believe our mirror to be in sync with the upstream"
                        iterations = 0
                        while iterations < options.maxretries:
                                if options.verbose:
                                        print "Sleeping for %d seconds" % options.wait
                                time.sleep(options.wait)
                                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                                        upstream_timestamp = getLastModifiedTime(upstream_trace)
                                        break
                                else:
                                        if options.verbose:
                                                print "Upstream trace file last modified on", upstream_timestamp
                                        iterations += 1
                        else:
                                if options.verbose and iterations > 0:
                                        print "Giving up"
                                sys.exit(1)
                        if options.verbose:
                                print "Upstream trace file last modified on", upstream_timestamp
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)

if __name__ == '__main__':
        main()
1	#!/usr/bin/python
2
3	import urllib2
4	import sys
5	import time
6	import stat
7	import os
8	from optparse import OptionParser
9
10	###
11	# Script to check if a local mirror is in sync with an HTTP-accessible remote
12	# mirror (based on the /debian/project/trace file)
13	#
14	# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
15	#
16	# Permission to use granted under the GNU General Public License v2
17
18	class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
19	"""The class you need to have to get useful IMS functionality"""
20
21	def http_error_default(self, req, fp, code, msg, headers):
22	"""Returns the result code of the transaction"""
23
24	result = urllib2.HTTPError(
25	req.get_full_url(), code, msg, headers, fp)
26	result.status = code
27	return result
28
29
30	def getLastModifiedTime(url):
31	"""Makes a request for url and returns the Last-Modified header"""
32
33	request = urllib2.Request(url)
34	opener = urllib2.build_opener()
35	data = opener.open(request)
36	return(data.headers.dict["last-modified"])
37
38
39	def URLisNewerThan(url, epoch):
40	"""Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""
41
42	request = urllib2.Request(url)
43	request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
44	opener = urllib2.build_opener(DefaultErrorHandler())
45	data = opener.open(request)
46	if (hasattr(data, 'status') and data.status == 304):
47	return False
48	else:
49	return True
50
51
52	def main():
53	parser = OptionParser()
54
55	parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
56	parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
57	help="Seconds to wait between checks for")
58	parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
59	help="Maximum number of interations before giving up")
60	parser.add_option("-v", "--verbose", action="store_true",
61	dest="verbose", default=False,
62	help="Print additional information about behaviour")
63
64	(options, args) = parser.parse_args()
65
66	if len(args) != 2:
67	parser.print_usage()
68	else:
69
70	UPSTREAM=args[0]
71	MIRROR_ROOT=args[1]
72	upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
73	local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)
74
75	#upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
76	upstream_timestamp = getLastModifiedTime(upstream_trace)
77
78	if options.verbose:
79	print "Upstream trace file last modified on", upstream_timestamp
80
81	local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]
82
83	if options.verbose:
84	print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))
85
86	if URLisNewerThan(upstream_trace, local_trace_timestamp):
87	if options.verbose:
88	print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
89	sys.exit(0)
90	else:
91	if options.verbose:
92	print "I believe our mirror to be in sync with the upstream"
93	iterations = 0
94	while iterations < options.maxretries:
95	if options.verbose:
96	print "Sleeping for %d seconds" % options.wait
97	time.sleep(options.wait)
98	if URLisNewerThan(upstream_trace, local_trace_timestamp):
99	upstream_timestamp = getLastModifiedTime(upstream_trace)
100	break
101	else:
102	if options.verbose:
103	print "Upstream trace file last modified on", upstream_timestamp
104	iterations += 1
105	else:
106	if options.verbose and iterations > 0:
107	print "Giving up"
108	sys.exit(1)
109	if options.verbose:
110	print "Upstream trace file last modified on", upstream_timestamp
111	print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
112	sys.exit(0)
113
114	if __name__ == '__main__':
115	main()