From 951f2d517fa2e464d654a54bebacbd983f944c62 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Jan 2014 01:57:37 +0100 Subject: change parameter names to reflect changes --- netlib/tcp.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index 33f7ef3a..d35818bf 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -138,8 +138,8 @@ class Reader(_FileLike): raise NetLibTimeout except socket.timeout: raise NetLibTimeout - except socket.error: - raise NetLibDisconnect + except socket.error, v: + raise NetLibDisconnect(v[1]) except SSL.SysCallError: raise NetLibDisconnect except SSL.Error, v: @@ -255,16 +255,13 @@ class BaseHandler: """ rbufsize = -1 wbufsize = -1 - def __init__(self, connection, client_address, server): + def __init__(self, connection): self.connection = connection self.rfile = Reader(self.connection.makefile('rb', self.rbufsize)) self.wfile = Writer(self.connection.makefile('wb', self.wbufsize)) - self.client_address = client_address - self.server = server self.finished = False self.ssl_established = False - self.clientcert = None def convert_to_ssl(self, cert, key, method=SSLv23_METHOD, options=None, handle_sni=None, request_client_cert=False, cipher_list=None): @@ -371,13 +368,13 @@ class TCPServer: self.port = self.server_address[1] self.socket.listen(self.request_queue_size) - def request_thread(self, request, client_address): + def connection_thread(self, connection, client_address): try: - self.handle_connection(request, client_address) - request.close() + self.handle_client_connection(connection, client_address) except: - self.handle_error(request, client_address) - request.close() + self.handle_error(connection, client_address) + finally: + connection.close() def serve_forever(self, poll_interval=0.1): self.__is_shut_down.clear() @@ -391,10 +388,10 @@ class TCPServer: else: raise if self.socket in r: - request, client_address = self.socket.accept() + connection, client_address = self.socket.accept() t = threading.Thread( - target = self.request_thread, - args = (request, client_address) + target = self.connection_thread, + args = (connection, client_address) ) t.setDaemon(1) t.start() @@ -410,7 +407,7 @@ class TCPServer: def handle_error(self, request, client_address, fp=sys.stderr): """ - Called when handle_connection raises an exception. + Called when handle_client_connection raises an exception. """ # If a thread has persisted after interpreter exit, the module might be # none. @@ -421,7 +418,7 @@ class TCPServer: print >> fp, exc print >> fp, '-'*40 - def handle_connection(self, request, client_address): # pragma: no cover + def handle_client_connection(self, conn, client_address): # pragma: no cover """ Called after client connection. """ -- cgit v1.2.3 From d0a6d2e2545089893d3789e3c787e269645df852 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Jan 2014 05:33:21 +0100 Subject: fix tests, remove duplicate code --- netlib/tcp.py | 91 ++++++++++++++++++++++++----------------------------------- 1 file changed, 37 insertions(+), 54 deletions(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index d35818bf..e48f4f6b 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -138,8 +138,8 @@ class Reader(_FileLike): raise NetLibTimeout except socket.timeout: raise NetLibTimeout - except socket.error, v: - raise NetLibDisconnect(v[1]) + except socket.error: + raise NetLibDisconnect except SSL.SysCallError: raise NetLibDisconnect except SSL.Error, v: @@ -173,7 +173,40 @@ class Reader(_FileLike): return result -class TCPClient: +class SocketCloseMixin: + def finish(self): + self.finished = True + try: + if not getattr(self.wfile, "closed", False): + self.wfile.flush() + self.close() + self.wfile.close() + self.rfile.close() + except (socket.error, NetLibDisconnect): + # Remote has disconnected + pass + + def close(self): + """ + Does a hard close of the socket, i.e. a shutdown, followed by a close. + """ + try: + if self.ssl_established: + self.connection.shutdown() + self.connection.sock_shutdown(socket.SHUT_WR) + else: + self.connection.shutdown(socket.SHUT_WR) + #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent. + #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html + while self.connection.recv(4096): + pass + self.connection.close() + except (socket.error, SSL.Error, IOError): + # Socket probably already closed + pass + + +class TCPClient(SocketCloseMixin): rbufsize = -1 wbufsize = -1 def __init__(self, host, port, source_address=None, use_ipv6=False): @@ -228,27 +261,8 @@ class TCPClient: def gettimeout(self): return self.connection.gettimeout() - def close(self): - """ - Does a hard close of the socket, i.e. a shutdown, followed by a close. - """ - try: - if self.ssl_established: - self.connection.shutdown() - self.connection.sock_shutdown(socket.SHUT_WR) - else: - self.connection.shutdown(socket.SHUT_WR) - #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent. - #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html - while self.connection.recv(4096): - pass - self.connection.close() - except (socket.error, SSL.Error, IOError): - # Socket probably already closed - pass - -class BaseHandler: +class BaseHandler(SocketCloseMixin): """ The instantiator is expected to call the handle() and finish() methods. @@ -315,43 +329,12 @@ class BaseHandler: self.rfile.set_descriptor(self.connection) self.wfile.set_descriptor(self.connection) - def finish(self): - self.finished = True - try: - if not getattr(self.wfile, "closed", False): - self.wfile.flush() - self.close() - self.wfile.close() - self.rfile.close() - except (socket.error, NetLibDisconnect): - # Remote has disconnected - pass - def handle(self): # pragma: no cover raise NotImplementedError def settimeout(self, n): self.connection.settimeout(n) - def close(self): - """ - Does a hard close of the socket, i.e. a shutdown, followed by a close. - """ - try: - if self.ssl_established: - self.connection.shutdown() - self.connection.sock_shutdown(socket.SHUT_WR) - else: - self.connection.shutdown(socket.SHUT_WR) - # Section 4.2.2.13 of RFC 1122 tells us that a close() with any - # pending readable data could lead to an immediate RST being sent. - # http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html - while self.connection.recv(4096): - pass - except (socket.error, SSL.Error): - # Socket probably already closed - pass - self.connection.close() class TCPServer: -- cgit v1.2.3 From 763cb90b66b23cd94b6e37df3d4c7b8e7f89492a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jan 2014 17:26:35 +0100 Subject: add tcp.Address to unify ipv4/ipv6 address handling --- netlib/tcp.py | 56 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index e48f4f6b..bad166d0 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -173,6 +173,35 @@ class Reader(_FileLike): return result +class Address(tuple): + """ + This class wraps an IPv4/IPv6 tuple to provide named attributes and ipv6 information. + """ + def __new__(cls, address, use_ipv6=False): + a = super(Address, cls).__new__(cls, tuple(address)) + a.family = socket.AF_INET6 if use_ipv6 else socket.AF_INET + return a + + @classmethod + def wrap(cls, t): + if isinstance(t, cls): + return t + else: + return cls(t) + + @property + def host(self): + return self[0] + + @property + def port(self): + return self[1] + + @property + def is_ipv6(self): + return self.family == socket.AF_INET6 + + class SocketCloseMixin: def finish(self): self.finished = True @@ -209,10 +238,9 @@ class SocketCloseMixin: class TCPClient(SocketCloseMixin): rbufsize = -1 wbufsize = -1 - def __init__(self, host, port, source_address=None, use_ipv6=False): - self.host, self.port = host, port + def __init__(self, address, source_address=None): + self.address = Address.wrap(address) self.source_address = source_address - self.use_ipv6 = use_ipv6 self.connection, self.rfile, self.wfile = None, None, None self.cert = None self.ssl_established = False @@ -245,14 +273,14 @@ class TCPClient(SocketCloseMixin): def connect(self): try: - connection = socket.socket(socket.AF_INET6 if self.use_ipv6 else socket.AF_INET, socket.SOCK_STREAM) + connection = socket.socket(self.address.family, socket.SOCK_STREAM) if self.source_address: connection.bind(self.source_address) - connection.connect((self.host, self.port)) + connection.connect(self.address) self.rfile = Reader(connection.makefile('rb', self.rbufsize)) self.wfile = Writer(connection.makefile('wb', self.wbufsize)) except (socket.error, IOError), err: - raise NetLibError('Error connecting to "%s": %s' % (self.host, err)) + raise NetLibError('Error connecting to "%s": %s' % (self.address[0], err)) self.connection = connection def settimeout(self, n): @@ -269,8 +297,9 @@ class BaseHandler(SocketCloseMixin): """ rbufsize = -1 wbufsize = -1 - def __init__(self, connection): + def __init__(self, connection, address): self.connection = connection + self.address = Address.wrap(address) self.rfile = Reader(self.connection.makefile('rb', self.rbufsize)) self.wfile = Writer(self.connection.makefile('wb', self.wbufsize)) @@ -339,19 +368,18 @@ class BaseHandler(SocketCloseMixin): class TCPServer: request_queue_size = 20 - def __init__(self, server_address, use_ipv6=False): - self.server_address = server_address - self.use_ipv6 = use_ipv6 + def __init__(self, address): + self.address = Address.wrap(address) self.__is_shut_down = threading.Event() self.__shutdown_request = False - self.socket = socket.socket(socket.AF_INET6 if self.use_ipv6 else socket.AF_INET, socket.SOCK_STREAM) + self.socket = socket.socket(self.address.family, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.socket.bind(self.server_address) - self.server_address = self.socket.getsockname() - self.port = self.server_address[1] + self.socket.bind(self.address) + self.address = Address.wrap(self.socket.getsockname()) self.socket.listen(self.request_queue_size) def connection_thread(self, connection, client_address): + client_address = Address(client_address) try: self.handle_client_connection(connection, client_address) except: -- cgit v1.2.3 From e18ac4b672e8645388dc8057801092ce417f1511 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jan 2014 20:30:16 +0100 Subject: re-add server attribute to BaseHandler --- netlib/tcp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index bad166d0..729e513e 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -297,9 +297,11 @@ class BaseHandler(SocketCloseMixin): """ rbufsize = -1 wbufsize = -1 - def __init__(self, connection, address): + + def __init__(self, connection, address, server): self.connection = connection self.address = Address.wrap(address) + self.server = server self.rfile = Reader(self.connection.makefile('rb', self.rbufsize)) self.wfile = Writer(self.connection.makefile('wb', self.wbufsize)) -- cgit v1.2.3 From ff9656be80192ac837cf98997f9fe6c00c9c5a32 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 30 Jan 2014 20:07:30 +0100 Subject: remove subclassing of tuple in tcp.Address, move StateObject into netlib --- netlib/tcp.py | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index 729e513e..c26d1191 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -1,6 +1,7 @@ import select, socket, threading, sys, time, traceback from OpenSSL import SSL import certutils +from netlib.stateobject import StateObject SSLv2_METHOD = SSL.SSLv2_METHOD SSLv3_METHOD = SSL.SSLv3_METHOD @@ -173,14 +174,13 @@ class Reader(_FileLike): return result -class Address(tuple): +class Address(StateObject): """ This class wraps an IPv4/IPv6 tuple to provide named attributes and ipv6 information. """ - def __new__(cls, address, use_ipv6=False): - a = super(Address, cls).__new__(cls, tuple(address)) - a.family = socket.AF_INET6 if use_ipv6 else socket.AF_INET - return a + def __init__(self, address, use_ipv6=False): + self.address = address + self.family = socket.AF_INET6 if use_ipv6 else socket.AF_INET @classmethod def wrap(cls, t): @@ -189,18 +189,35 @@ class Address(tuple): else: return cls(t) + def __call__(self): + return self.address + @property def host(self): - return self[0] + return self.address[0] @property def port(self): - return self[1] + return self.address[1] @property - def is_ipv6(self): + def use_ipv6(self): return self.family == socket.AF_INET6 + def _load_state(self, state): + self.address = state["address"] + self.family = socket.AF_INET6 if state["use_ipv6"] else socket.AF_INET + + def _get_state(self): + return dict( + address=self.address, + use_ipv6=self.use_ipv6 + ) + + @classmethod + def _from_state(cls, state): + return cls(**state) + class SocketCloseMixin: def finish(self): @@ -240,7 +257,7 @@ class TCPClient(SocketCloseMixin): wbufsize = -1 def __init__(self, address, source_address=None): self.address = Address.wrap(address) - self.source_address = source_address + self.source_address = Address.wrap(source_address) if source_address else None self.connection, self.rfile, self.wfile = None, None, None self.cert = None self.ssl_established = False @@ -275,12 +292,12 @@ class TCPClient(SocketCloseMixin): try: connection = socket.socket(self.address.family, socket.SOCK_STREAM) if self.source_address: - connection.bind(self.source_address) - connection.connect(self.address) + connection.bind(self.source_address()) + connection.connect(self.address()) self.rfile = Reader(connection.makefile('rb', self.rbufsize)) self.wfile = Writer(connection.makefile('wb', self.wbufsize)) except (socket.error, IOError), err: - raise NetLibError('Error connecting to "%s": %s' % (self.address[0], err)) + raise NetLibError('Error connecting to "%s": %s' % (self.address.host, err)) self.connection = connection def settimeout(self, n): @@ -376,7 +393,7 @@ class TCPServer: self.__shutdown_request = False self.socket = socket.socket(self.address.family, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.socket.bind(self.address) + self.socket.bind(self.address()) self.address = Address.wrap(self.socket.getsockname()) self.socket.listen(self.request_queue_size) @@ -427,7 +444,7 @@ class TCPServer: if traceback: exc = traceback.format_exc() print >> fp, '-'*40 - print >> fp, "Error in processing of request from %s:%s"%client_address + print >> fp, "Error in processing of request from %s:%s" % (client_address.host, client_address.port) print >> fp, exc print >> fp, '-'*40 -- cgit v1.2.3 From dc45b4bf19bff5edc0b72ccb68fad04d479aff83 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 31 Jan 2014 01:06:53 +0100 Subject: move StateObject back into libmproxy --- netlib/tcp.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index c26d1191..346bc053 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -1,7 +1,6 @@ import select, socket, threading, sys, time, traceback from OpenSSL import SSL import certutils -from netlib.stateobject import StateObject SSLv2_METHOD = SSL.SSLv2_METHOD SSLv3_METHOD = SSL.SSLv3_METHOD @@ -174,13 +173,13 @@ class Reader(_FileLike): return result -class Address(StateObject): +class Address(object): """ This class wraps an IPv4/IPv6 tuple to provide named attributes and ipv6 information. """ def __init__(self, address, use_ipv6=False): self.address = address - self.family = socket.AF_INET6 if use_ipv6 else socket.AF_INET + self.use_ipv6 = use_ipv6 @classmethod def wrap(cls, t): @@ -204,19 +203,9 @@ class Address(StateObject): def use_ipv6(self): return self.family == socket.AF_INET6 - def _load_state(self, state): - self.address = state["address"] - self.family = socket.AF_INET6 if state["use_ipv6"] else socket.AF_INET - - def _get_state(self): - return dict( - address=self.address, - use_ipv6=self.use_ipv6 - ) - - @classmethod - def _from_state(cls, state): - return cls(**state) + @use_ipv6.setter + def use_ipv6(self, b): + self.family = socket.AF_INET6 if b else socket.AF_INET class SocketCloseMixin: -- cgit v1.2.3 From 0bbc40dc33dd7bd3729e639874882dd6dd7ea818 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 4 Feb 2014 04:51:41 +0100 Subject: store used sni in TCPClient, add equality check for tcp.Address --- netlib/tcp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index 346bc053..94ea8806 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -207,8 +207,12 @@ class Address(object): def use_ipv6(self, b): self.family = socket.AF_INET6 if b else socket.AF_INET + def __eq__(self, other): + other = Address.wrap(other) + return (self.address, self.family) == (other.address, other.family) -class SocketCloseMixin: + +class SocketCloseMixin(object): def finish(self): self.finished = True try: @@ -250,6 +254,7 @@ class TCPClient(SocketCloseMixin): self.connection, self.rfile, self.wfile = None, None, None self.cert = None self.ssl_established = False + self.sni = None def convert_to_ssl(self, cert=None, sni=None, method=TLSv1_METHOD, options=None): """ @@ -267,6 +272,7 @@ class TCPClient(SocketCloseMixin): self.connection = SSL.Connection(context, self.connection) self.ssl_established = True if sni: + self.sni = sni self.connection.set_tlsext_host_name(sni) self.connection.set_connect_state() try: -- cgit v1.2.3 From 7fc544bc7ff8fd610ba9db92c0d3b59a0b040b5b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 5 Feb 2014 21:34:14 +0100 Subject: adjust netlib.wsgi to reflect changes in mitmproxys flow format --- netlib/tcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'netlib/tcp.py') diff --git a/netlib/tcp.py b/netlib/tcp.py index 94ea8806..34e47999 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -178,7 +178,7 @@ class Address(object): This class wraps an IPv4/IPv6 tuple to provide named attributes and ipv6 information. """ def __init__(self, address, use_ipv6=False): - self.address = address + self.address = tuple(address) self.use_ipv6 = use_ipv6 @classmethod -- cgit v1.2.3