svn.andrew.net.au/scripts/ready2mirror.py

#!/usr/bin/python

import urllib2
import sys
import time
import stat
import os
from optparse import OptionParser

###
# Script to check if a local mirror is in sync with an HTTP-accessible remote
# mirror (based on the /debian/project/trace file)
#
# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
#
# Permission to use granted under the GNU General Public License v2

class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
        """The class you need to have to get useful IMS functionality"""

        def http_error_default(self, req, fp, code, msg, headers):
                """Returns the result code of the transaction"""

                result = urllib2.HTTPError(
                        req.get_full_url(), code, msg, headers, fp)
                result.status = code
                return result


class Request2(urllib2.Request):

        def __init__(self, url, data=None, headers={},
                        origin_req_host=None, unverifiable=False, method="GET"):

                urllib2.Request(self, url, data, headers, origin_req_host, unverifiable)
                self.method = method

        def get_method(self):
                if self.has_data():
                        return "POST"
                else:
                        return self.method


def getLastModifiedTime(url):
        """Makes a request for url and returns the Last-Modified header"""

        request = Request2(url)
        opener = urllib2.build_opener()
        data = opener.open(request)
        return(data.headers.dict["last-modified"])


def URLisNewerThan(url, epoch):
        """Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""

        request = Request2(url)
        request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
        opener = urllib2.build_opener(DefaultErrorHandler())
        data = opener.open(request)
        if (hasattr(data, 'status') and data.status == 304):
                return False
        else:
                return True

def isSyncing():
        """Returns if the upstream mirror has a lock file indicating it is currently syncing with its upstream"""

        pass

def main():
        parser = OptionParser()

        parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
        parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
                help="Seconds to wait between checks for")
        parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
                help="Maximum number of interations before giving up")
        parser.add_option("-v", "--verbose", action="store_true", 
                dest="verbose", default=False,
                help="Print additional information about behaviour")

        (options, args) = parser.parse_args()

        if len(args) != 2:
                parser.print_usage()
        else:
        
                UPSTREAM=args[0]
                MIRROR_ROOT=args[1]
                upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
                local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)

                #upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
                upstream_timestamp = getLastModifiedTime(upstream_trace)

                if options.verbose:
                        print "Upstream trace file last modified on", upstream_timestamp

                local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]

                if options.verbose:
                        print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))

                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                        if options.verbose:
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)
                else:
                        if options.verbose:
                                print "I believe our mirror to be in sync with the upstream"
                        iterations = 0
                        while iterations < options.maxretries:
                                if options.verbose:
                                        print "Sleeping for %d seconds" % options.wait
                                time.sleep(options.wait)
                                if URLisNewerThan(upstream_trace, local_trace_timestamp):
                                        upstream_timestamp = getLastModifiedTime(upstream_trace)
                                        break
                                else:
                                        if options.verbose:
                                                print "Upstream trace file last modified on", upstream_timestamp
                                        iterations += 1
                        else:
                                if options.verbose and iterations > 0:
                                        print "Giving up"
                                sys.exit(1)
                        if options.verbose:
                                print "Upstream trace file last modified on", upstream_timestamp
                                print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
                        sys.exit(0)

if __name__ == '__main__':
        main()
1	#!/usr/bin/python
2
3	import urllib2
4	import sys
5	import time
6	import stat
7	import os
8	from optparse import OptionParser
9
10	###
11	# Script to check if a local mirror is in sync with an HTTP-accessible remote
12	# mirror (based on the /debian/project/trace file)
13	#
14	# Copyright (c) 2006 Andrew Pollock <me@andrew.net.au>
15	#
16	# Permission to use granted under the GNU General Public License v2
17
18	class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
19	"""The class you need to have to get useful IMS functionality"""
20
21	def http_error_default(self, req, fp, code, msg, headers):
22	"""Returns the result code of the transaction"""
23
24	result = urllib2.HTTPError(
25	req.get_full_url(), code, msg, headers, fp)
26	result.status = code
27	return result
28
29
30	class Request2(urllib2.Request):
31
32	def __init__(self, url, data=None, headers={},
33	origin_req_host=None, unverifiable=False, method="GET"):
34
35	urllib2.Request(self, url, data, headers, origin_req_host, unverifiable)
36	self.method = method
37
38	def get_method(self):
39	if self.has_data():
40	return "POST"
41	else:
42	return self.method
43
44
45	def getLastModifiedTime(url):
46	"""Makes a request for url and returns the Last-Modified header"""
47
48	request = Request2(url)
49	opener = urllib2.build_opener()
50	data = opener.open(request)
51	return(data.headers.dict["last-modified"])
52
53
54	def URLisNewerThan(url, epoch):
55	"""Makes an IMS request for url with epoch as the modification time and returns if URL has changed"""
56
57	request = Request2(url)
58	request.add_header('If-Modified-Since', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(epoch)))
59	opener = urllib2.build_opener(DefaultErrorHandler())
60	data = opener.open(request)
61	if (hasattr(data, 'status') and data.status == 304):
62	return False
63	else:
64	return True
65
66	def isSyncing():
67	"""Returns if the upstream mirror has a lock file indicating it is currently syncing with its upstream"""
68
69	pass
70
71	def main():
72	parser = OptionParser()
73
74	parser.set_description("Program to check if a local Debian mirror is out of sync with a remote Debian mirror")
75	parser.add_option("-w", "--wait-time", dest="wait", default=300, type="int",
76	help="Seconds to wait between checks for")
77	parser.add_option("-m", "--max-retry", dest="maxretries", default=0, type="int",
78	help="Maximum number of interations before giving up")
79	parser.add_option("-v", "--verbose", action="store_true",
80	dest="verbose", default=False,
81	help="Print additional information about behaviour")
82
83	(options, args) = parser.parse_args()
84
85	if len(args) != 2:
86	parser.print_usage()
87	else:
88
89	UPSTREAM=args[0]
90	MIRROR_ROOT=args[1]
91	upstream_trace = "".join(['http://', UPSTREAM, '/debian/project/trace/', UPSTREAM])
92	local_trace = os.path.join(MIRROR_ROOT, "debian", "project", "trace", UPSTREAM)
93
94	#upstream_timestamp = calendar.timegm(time.strptime(getLastModifiedTime(upstream_trace), "%a, %d %b %Y %H:%M:%S %Z"))
95	upstream_timestamp = getLastModifiedTime(upstream_trace)
96
97	if options.verbose:
98	print "Upstream trace file last modified on", upstream_timestamp
99
100	local_trace_timestamp = os.stat(local_trace)[stat.ST_MTIME]
101
102	if options.verbose:
103	print "Our copy of trace file last modified on", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(local_trace_timestamp))
104
105	if URLisNewerThan(upstream_trace, local_trace_timestamp):
106	if options.verbose:
107	print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
108	sys.exit(0)
109	else:
110	if options.verbose:
111	print "I believe our mirror to be in sync with the upstream"
112	iterations = 0
113	while iterations < options.maxretries:
114	if options.verbose:
115	print "Sleeping for %d seconds" % options.wait
116	time.sleep(options.wait)
117	if URLisNewerThan(upstream_trace, local_trace_timestamp):
118	upstream_timestamp = getLastModifiedTime(upstream_trace)
119	break
120	else:
121	if options.verbose:
122	print "Upstream trace file last modified on", upstream_timestamp
123	iterations += 1
124	else:
125	if options.verbose and iterations > 0:
126	print "Giving up"
127	sys.exit(1)
128	if options.verbose:
129	print "Upstream trace file last modified on", upstream_timestamp
130	print "Because the upstream trace file is newer than what is on the mirror, I think it's time to resync the mirror"
131	sys.exit(0)
132
133	if __name__ == '__main__':
134	main()