Welcome, guest | Sign In | My Account | Store | Cart

Server that runs a threaded, documenting XMLRCP server that uses HTTPS for transporting XML data.

(This code borrow's heavily from Laszlo Nagy's: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496786)

Python, 265 lines
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
#! /usr/bin/env python

"""
 *******************************************************************************
 * 
 * $Id: SecureDocXMLRPCServer.py 4 2008-06-04 18:44:13Z yingera $
 * $URL: https://xxxxxx/repos/utils/trunk/tools/SVNRPCServer.py $
 *
 * $Date: 2008-06-04 13:44:13 -0500 (Wed, 04 Jun 2008) $
 * $Author: yingera $
 *
 * Authors: Laszlo Nagy, Andrew Yinger
 *
 * Description: Threaded, Documenting SecureDocXMLRPCServer.py - over HTTPS.
 *
 *  requires pyOpenSSL: http://sourceforge.net/project/showfiles.php?group_id=31249
 *   ...and open SSL certs installed.
 *
 * Based on this article: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81549
 *
 *******************************************************************************
"""

import SocketServer
import BaseHTTPServer
import SimpleHTTPServer
import SimpleXMLRPCServer

import socket, os
from OpenSSL import SSL
from threading import Event, currentThread, Thread, Condition
from thread import start_new_thread as start
from DocXMLRPCServer import DocXMLRPCServer, DocXMLRPCRequestHandler

# static stuff
DEFAULTKEYFILE='key.pem'    # Replace with your PEM formatted key file
DEFAULTCERTFILE='certificate.crt'  # Replace with your PEM formatted certificate file


class SecureDocXMLRpcRequestHandler(DocXMLRPCRequestHandler):
    """Secure Doc XML-RPC request handler class.
    It it very similar to DocXMLRPCRequestHandler but it uses HTTPS for transporting XML data.
    """
    def setup(self):
        self.connection = self.request
        self.rfile = socket._fileobject(self.request, "rb", self.rbufsize)
        self.wfile = socket._fileobject(self.request, "wb", self.wbufsize)

    def address_string(self):
        "getting 'FQDN' from host seems to stall on some ip addresses, so... just (quickly!) return raw host address"
        host, port = self.client_address
        #return socket.getfqdn(host)
        return host

    def do_POST(self):
        """Handles the HTTPS POST request.
        It was copied out from SimpleXMLRPCServer.py and modified to shutdown the socket cleanly.
        """
        try:
            # get arguments
            data = self.rfile.read(int(self.headers["content-length"]))
            # In previous versions of SimpleXMLRPCServer, _dispatch
            # could be overridden in this class, instead of in
            # SimpleXMLRPCDispatcher. To maintain backwards compatibility,
            # check to see if a subclass implements _dispatch and dispatch
            # using that method if present.
            response = self.server._marshaled_dispatch(data, getattr(self, '_dispatch', None))
        except: # This should only happen if the module is buggy
            # internal error, report as HTTP server error
            self.send_response(500)
            self.end_headers()
        else:
            # got a valid XML RPC response
            self.send_response(200)
            self.send_header("Content-type", "text/xml")
            self.send_header("Content-length", str(len(response)))
            self.end_headers()
            self.wfile.write(response)

            # shut down the connection
            self.wfile.flush()
            self.connection.shutdown() # Modified here!

    def do_GET(self):
        """Handles the HTTP GET request.

        Interpret all HTTP GET requests as requests for server
        documentation.
        """
        # Check that the path is legal
        if not self.is_rpc_path_valid():
            self.report_404()
            return

        response = self.server.generate_html_documentation()
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.send_header("Content-length", str(len(response)))
        self.end_headers()
        self.wfile.write(response)

        # shut down the connection
        self.wfile.flush()
        self.connection.shutdown() # Modified here!

    def report_404 (self):
        # Report a 404 error
        self.send_response(404)
        response = 'No such page'
        self.send_header("Content-type", "text/plain")
        self.send_header("Content-length", str(len(response)))
        self.end_headers()
        self.wfile.write(response)
        # shut down the connection
        self.wfile.flush()
        self.connection.shutdown() # Modified here!



class CustomThreadingMixIn:
    """Mix-in class to handle each request in a new thread."""
    # Decides how threads will act upon termination of the main process
    daemon_threads = True

    def process_request_thread(self, request, client_address):
        """Same as in BaseServer but as a thread.
        In addition, exception handling is done here.
        """
        try:
            self.finish_request(request, client_address)
            self.close_request(request)
        except (socket.error, SSL.SysCallError), why:
            print 'socket.error finishing request from "%s"; Error: %s' % (client_address, str(why))
            self.close_request(request)
        except:
            self.handle_error(request, client_address)
            self.close_request(request)

    def process_request(self, request, client_address):
        """Start a new thread to process the request."""
        t = Thread(target = self.process_request_thread, args = (request, client_address))
        if self.daemon_threads:
            t.setDaemon(1)
        t.start()



class SecureDocXMLRPCServer(CustomThreadingMixIn, DocXMLRPCServer):
    def __init__(self, registerInstance, server_address, keyFile=DEFAULTKEYFILE, certFile=DEFAULTCERTFILE, logRequests=True):
        """Secure Documenting XML-RPC server.
        It it very similar to DocXMLRPCServer but it uses HTTPS for transporting XML data.
        """
        DocXMLRPCServer.__init__(self, server_address, SecureDocXMLRpcRequestHandler, logRequests)
        self.logRequests = logRequests

        # stuff for doc server
        try: self.set_server_title(registerInstance.title)
        except AttributeError: self.set_server_title('default title')
        try: self.set_server_name(registerInstance.name)
        except AttributeError: self.set_server_name('default name')
        if registerInstance.__doc__: self.set_server_documentation(registerInstance.__doc__)
        else: self.set_server_documentation('default documentation')
        self.register_introspection_functions()

        # init stuff, handle different versions:
        try:
            SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self)
        except TypeError:
            # An exception is raised in Python 2.5 as the prototype of the __init__
            # method has changed and now has 3 arguments (self, allow_none, encoding)
            SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, False, None)
        SocketServer.BaseServer.__init__(self, server_address, SecureDocXMLRpcRequestHandler)
        self.register_instance(registerInstance) # for some reason, have to register instance down here!

        # SSL socket stuff
        ctx = SSL.Context(SSL.SSLv23_METHOD)
        ctx.use_privatekey_file(keyFile)
        ctx.use_certificate_file(certFile)
        self.socket = SSL.Connection(ctx, socket.socket(self.address_family, self.socket_type))
        self.server_bind()
        self.server_activate()

        # requests count and condition, to allow for keyboard quit via CTL-C
        self.requests = 0
        self.rCondition = Condition()


    def startup(self):
        'run until quit signaled from keyboard...'
        print 'server starting; hit CTRL-C to quit...'
        while True:
            try:
                self.rCondition.acquire()
                start(self.handle_request, ()) # we do this async, because handle_request blocks!
                while not self.requests:
                    self.rCondition.wait(timeout=3.0)
                if self.requests: self.requests -= 1
                self.rCondition.release()
            except KeyboardInterrupt:
                print "quit signaled, i'm done."
                return

    def get_request(self):
        request, client_address = self.socket.accept()
        self.rCondition.acquire()
        self.requests += 1
        self.rCondition.notifyAll()
        self.rCondition.release()
        return (request, client_address)

    def listMethods(self):
        'return list of method names (strings)'
        methodNames = self.funcs.keys()
        methodNames.sort()
        return methodNames

    def methodHelp(self, methodName):
        'method help'
        if methodName in self.funcs:
            return self.funcs[methodName].__doc__
        else:
            raise Exception('method "%s" is not supported' % methodName)



def TestSampleServer():
    """Test xml rpc over https server"""
    class ExampleRegisters:
        '''just some test methods to try out new secure doc xml-rpc server...'''
        title = 'test server methods'
        name = 'test server'
        def __init__(self):
            import string
            self.python_string = string
            
        def add(self, x, y):
            return x + y
    
        def mult(self,x,y):
            return x*y
    
        def div(self,x,y):
            return x//y

        def poop(self): return 'poop'
        
    server_address = (socket.gethostname(), 9779) # (address, port)
    server = SecureDocXMLRPCServer(ExampleRegisters(), server_address, DEFAULTKEYFILE, DEFAULTCERTFILE)    
    sa = server.socket.getsockname()
    print "Serving HTTPS on", sa[0], "port", sa[1]
    server.startup()


if __name__ == '__main__':
    TestSampleServer()


# Here is an xmlrpc client for testing:
"""
import xmlrpclib

server = xmlrpclib.Server('https://localhost:9777')
print server.add(1,2)
print server.div(10,4)
"""

I saw a need for a threaded XMLRPC server that combines secure connections with XML RPC, and is self-documenting. This code combines DocXMLRPCServer with Laszlo Nagy's SecureXMLRPCServer.

It also has these additional features: - the server works with python versions 2.3 - 2.5.2 - the server is threaded, so each client request is handled in its own thread - the server's threaded request handler does exception handling properly - the server can be cleanly terminated, by issueing 'CTRL-C' at keyboard - the server has an address lookup speedup - the server is self-documenting (it's remote methods may be viewed in a browser)

You will need to install pyOpenSSL: http://sourceforge.net/project/showfiles.php?group_id=31249

You will also need to use OpenSSL to generate your own (self-signed) key/certificate pair for your server:

(assuming openssl is installed, and in your path somehow...) openssl genrsa -out key.pem 1024 openssl req -new -key key.pem -out request.pem openssl x509 -req -days 1825 -in request.pem -signkey key.pem -out certificate.crt

This server has been extensively tested on windows, but should work on other platforms as well.

3 comments

Julian 11 years, 8 months ago  # | flag

Hi There Andrew, great work is this. I got all set up to use it and then realised what a doc server was :) I was wondering if you could strip out the doc server and just have a Threaded SSL XMLRPC Server. I've been looking at dumping your threading code into the Laszlo recipe, but I can't make it work, I don't really understand it that well. Could you help me with it, maybe create a new recipe.

Keith Briggs 10 years, 2 months ago  # | flag

There is an error here as the server uses port 9779 and the client uses port 9777. Even after fixing this, I get (with python 2.7.1):

kbriggs@gold:~/python> python ./threaded_XMLRPC_server.py Serving HTTPS on 127.0.0.1 port 9779 server starting; hit CTRL-C to quit...

127.0.0.1 - - [07/Sep/2011 13:59:54] "POST /RPC2 HTTP/1.1" 200 -

Exception happened during processing of request from ('127.0.0.1', 51070) Traceback (most recent call last): File "./threaded_XMLRPC_server.py", line 135, in process_request_thread self.finish_request(request, client_address) File "/usr/local/lib/python2.7/SocketServer.py", line 323, in finish_request self.RequestHandlerClass(request, client_address, self) File "/usr/local/lib/python2.7/SocketServer.py", line 641, in __init__ self.finish() File "/usr/local/lib/python2.7/SocketServer.py", line 694, in finish self.wfile.flush() File "/usr/local/lib/python2.7/socket.py", line 303, in flush self._sock.sendall(view[write_offset:write_offset+buffer_size])

TypeError: must be string or read-only buffer, not memoryview
127.0.0.1 - - [07/Sep/2011 13:59:54] "POST /RPC2 HTTP/1.1" 200 -

Exception happened during processing of request from ('127.0.0.1', 51071) Traceback (most recent call last): File "./threaded_XMLRPC_server.py", line 135, in process_request_thread self.finish_request(request, client_address) File "/usr/local/lib/python2.7/SocketServer.py", line 323, in finish_request self.RequestHandlerClass(request, client_address, self) File "/usr/local/lib/python2.7/SocketServer.py", line 641, in __init__ self.finish() File "/usr/local/lib/python2.7/SocketServer.py", line 694, in finish self.wfile.flush() File "/usr/local/lib/python2.7/socket.py", line 303, in flush self._sock.sendall(view[write_offset:write_offset+buffer_size])

TypeError: must be string or read-only buffer, not memoryview
Andrew Yinger (author) 9 years, 11 months ago  # | flag

Keith:

i finally ran into this same error, after upgrading to python 2.7.2.

i fixed by using .13 version of pyOpenSSL, which can be found here:

http://pypi.python.org/pypi/pyOpenSSL

hope that works for you too!