diff --git a/cachebrowser/api.py b/cachebrowser/api.py index 4ab95b4..b875e81 100644 --- a/cachebrowser/api.py +++ b/cachebrowser/api.py @@ -51,7 +51,8 @@ def on_request(self, env, start_response): request[query] = request[query][0] else: inp = env.get('wsgi.input', None) - request = json.loads(inp.read()) if inp is not None else {} + inp_val = inp.read() if inp is not None else '' + request = json.loads(inp_val) if len(inp_val) > 0 else {} response = self.method_handlers[method][path](request) @@ -84,6 +85,7 @@ def __init__(self, *args, **kwargs): self.register_api('PUT', '/host/bootstrap', self.action_add_host) self.register_api('GET', '/host/check', self.action_check_host) self.register_api('GET', '/cachebrowse', self.action_get) + self.register_api('POST', '/proxy/https/clear', self.clear_https_proxy) @staticmethod def action_add_host(request): @@ -136,3 +138,12 @@ def action_get(request): return response.get_raw(), ResponseOptions(send_json=False) else: return response.body, ResponseOptions(send_json=False) + + @staticmethod + def clear_https_proxy(request): + from cachebrowser import common + rack = common.context.get('server_rack') + server = rack.get_server('proxy-https') + + server.close_all() + return {'result': 'success'} \ No newline at end of file diff --git a/cachebrowser/cli.py b/cachebrowser/cli.py index 98ea38c..938fe14 100644 --- a/cachebrowser/cli.py +++ b/cachebrowser/cli.py @@ -86,6 +86,7 @@ def __init__(self, *args, **kwargs): self.register_command('list hosts', self.domain_list) self.register_command('list cdn', self.cdn_list) self.register_command('get', self.make_request) + self.register_command('https clear', self.close_https_proxy) def domain_add(self, hostname=None): """ @@ -124,6 +125,14 @@ def make_request(self, url, target=None, *args): response = http.request(url, target=target) self.send_line(response.read()) + def close_https_proxy(self): + from cachebrowser import common + rack = common.context.get('server_rack') + server = rack.get_server('proxy-https') + + server.close_all() + self.send_line("Closed all https-proxy connections") + class UnrecognizedCommandException(Exception): def __init__(self, command, valid_commands=None): diff --git a/cachebrowser/extensions/__init__.py b/cachebrowser/extensions/__init__.py index bc469c5..17ee2aa 100644 --- a/cachebrowser/extensions/__init__.py +++ b/cachebrowser/extensions/__init__.py @@ -1 +1 @@ -# import fingerprint +import fingerprint diff --git a/cachebrowser/extensions/fingerprint.py b/cachebrowser/extensions/fingerprint.py new file mode 100644 index 0000000..c10d1cc --- /dev/null +++ b/cachebrowser/extensions/fingerprint.py @@ -0,0 +1,180 @@ +import json +import urlparse + +from cachebrowser import common, http, dns +import sys +import time +from datetime import datetime + +# +# handlers = set() +# +# def request_wrapper(request): +# def inner(url, **kwargs): +# parsed_url = urlparse.urlparse(url) +# target, cachebrowsed = dns.resolve_host(parsed_url.hostname, use_cachebrowser_db=False) +# kwargs['target'] = target +# kwargs['cachebrowse'] = False +# +# for handler in handlers: +# handler.log(url, target) +# +# return request(url, **kwargs) +# return inner +# +# +# # Wrap request method +# http.request = request_wrapper(http.request) +from cachebrowser.proxybeta import proxy_logger + +rack = common.context.get('server_rack') +cli_server = rack.get_server('cli') +cli_handler_class = cli_server.handler + +api_server = rack.get_server('api') +api_handler_class = api_server.handler + + +class Recorder(): + def __init__(self): + self.logging = False + self.data_sets = None + self.targets = [] + + def log(self, host, target, scheme, url, cachebrowsed, **kwargs): + if not self.logging: + return + self.targets.append({ + 'host': host, + 'url': url, + 'ip': target, + 'scheme': scheme, + 'cachebrowsed': cachebrowsed + }) + + def start_recording(self): + self.logging = True + self.targets = [] + self.data_sets = None + + def stop_recording(self): + self.logging = False + data = {'timestamp': int(time.time()), 'date': str(datetime.now()), 'connections': self.targets} + self.data_sets = data + + def results(self): + return self.data_sets + + +recorder = Recorder() +proxy_logger.register_handler(recorder.log) +# handlers.add(recorder.log) + + +class CLIHandler(cli_handler_class): + def __init__(self, *args, **kwargs): + super(CLIHandler, self).__init__(*args, **kwargs) + self.register_command('fp start', self.fp_start) + self.register_command('fp stop', self.fp_stop) + self.register_command('fp save', self.fp_save) + # self.register_command('fp run', self.fp_auto_run) + self.logging = False + self.data_sets = [] + self.targets = [] + + def on_connect(self, *args, **kwargs): + super(CLIHandler, self).on_connect(*args, **kwargs) + handlers.add(self) + + def on_close(self, *args, **kwargs): + super(CLIHandler, self).on_close(*args, **kwargs) + handlers.remove(self) + + def log(self, url, target): + if not self.logging: + return + self.targets.append({ + 'url': url, + 'ip': target + }) + + def fp_start(self): + self.logging = True + self.targets = [] + self.data_sets = [] + + def fp_stop(self, host): + self.logging = False + + data = { + 'host': host, + 'timestamp': time.time() + } + + for target in self.targets: + from ipwhois import IPWhois + try: + whois = IPWhois(target['ip']) + name = whois.lookup_rdap()['network']['name'] + target['org'] = name + except: + print("EXCEPTION") + sys.exit(1) + + data['connections'] = self.targets + self.data_sets.append(data) + self.send_line("Saved as %s" % host) + + def fp_save(self, push_server='http://www.cachebrowser.info/fp/', **kwargs): + import requests + + url = "%s/records/" % push_server + for dataset in self.data_sets: + data = dataset + for key in kwargs: + data[key] = kwargs[key] + + if data.has_key('tags'): + data['tags'] = data['tags'].split(',') + + data_json = json.dumps(data) + headers = {'Content-type': 'application/json'} + response = requests.post(url, data=data_json, headers=headers) + if response.status_code == 200: + self.send_line("Posted %s to server" % data['host']) + else: + self.send_line("Saving %s failed: %d %s" % (data['host'], response.status_code, response.reason)) + + +class APIHandler(api_handler_class): + def __init__(self, *args, **kwargs): + super(APIHandler, self).__init__(*args, **kwargs) + self.register_api('POST', '/record/start', self.fp_start) + self.register_api('POST', '/record/stop', self.fp_stop) + self.register_api('GET', '/record/results', self.fp_results) + + def fp_start(self, message): + try: + recorder.start_recording() + return {'status': 'ok'} + except Exception as e: + return {'status': 'fail', 'error': e.message} + + def fp_stop(self, message): + try: + recorder.stop_recording() + return {'status': 'ok'} + except Exception as e: + return {'status': 'fail', 'error': e.message} + + def fp_results(self, message): + try: + results = recorder.results() + return {'status': 'ok', 'results': results} + except Exception as e: + return {'status': 'fail', 'error': e.message} + +cli_server.handler = CLIHandler +api_server.handler = APIHandler + + diff --git a/cachebrowser/main.py b/cachebrowser/main.py index 4cb2b0b..d37ccad 100644 --- a/cachebrowser/main.py +++ b/cachebrowser/main.py @@ -49,6 +49,11 @@ def run_cachebrowser(): rack.create_server('proxy', port=8080, handler=proxy.ProxyConnection) rack.create_server('http', port=9005, handler=http.HttpConnection) + + from cachebrowser import proxybeta + rack.add_server('proxybeta', HttpServer(port=8090, handler=proxybeta.ProxyHandler)) + rack.create_server('proxy-https', port=8095, handler=proxybeta.SSLProxyHandler) + common.context['server_rack'] = rack load_extensions() diff --git a/cachebrowser/network.py b/cachebrowser/network.py index d85b801..6150cf2 100644 --- a/cachebrowser/network.py +++ b/cachebrowser/network.py @@ -1,5 +1,6 @@ import inspect import socket +import errno import gevent import gevent.pywsgi from gevent.server import StreamServer @@ -43,18 +44,32 @@ def __init__(self, ip, port, handler=None): self.server = StreamServer((ip, port), self._handle) self.handler = handler + self.active_connections = set() + def start(self): return self.server.start() def stop(self): self.server.stop() + def close_all(self): + for connection in self.active_connections: + connection.close() + self.active_connections.clear() + + def add_active_connection(self, connection): + self.active_connections.add(connection) + + def remove_active_connection(self, connection): + if connection in self.active_connections: + self.active_connections.remove(connection) + def _handle(self, sock, address): if self.handler is None: return if inspect.isclass(self.handler): - return self.handler().loop(sock, address=address) + return self.handler(server=self).loop(sock, address=address) else: return self.handler.loop(sock, address=address) @@ -65,6 +80,7 @@ def __init__(self, *args, **kwargs): self.address = None self.alive = True + self._server = kwargs.get('server', None) self._read_size = 1024 def loop(self, sock, address): @@ -72,6 +88,9 @@ def loop(self, sock, address): self.address = address self.alive = True + if self._server: + self._server.add_active_connection(self) + self.on_connect(sock, address) self._loop_on_socket() @@ -88,7 +107,15 @@ def _loop_on_socket(self): self.on_data(buff) except socket.error as e: self.alive = False - self.on_error(e) + + # If error is bad file descriptor its probably because the socket was closed + if e.errno == errno.EBADF: + self.on_close() + else: + self.on_error(e) + + if self._server: + self._server.remove_active_connection(self) def send(self, data): self.socket.send(data) @@ -134,7 +161,7 @@ def _handle(self, env, start_response): else: return self.handler.on_request(env, start_response) - +gevent.pywsgi.WSGIHandler class HttpConnectionHandler(object): def on_request(self, env, start_response): pass \ No newline at end of file diff --git a/cachebrowser/proxy.py b/cachebrowser/proxy.py index 541de56..c05ab66 100644 --- a/cachebrowser/proxy.py +++ b/cachebrowser/proxy.py @@ -28,10 +28,13 @@ def on_data(self, data): def on_connect(self, sock, address): self._local_socket = sock + print("CONNECT") # logger.debug("New proxy connection established with %s" % str(self.address)) @silent_fail(log=True) def on_local_data(self, data): + print("LOCAL DATA %d" % len(data)) + print(data) if len(data) == 0: return @@ -46,6 +49,8 @@ def on_local_data(self, data): @silent_fail(log=True) def on_remote_data(self, data): + print("REMOTE DATA %d" % len(data)) + print(data) if len(data) == 0: return @@ -54,6 +59,7 @@ def on_remote_data(self, data): @silent_fail(log=True) def on_local_closed(self): + print("LOCAL CLOSED") if self._remote_socket is None: return @@ -64,12 +70,14 @@ def on_local_closed(self): @silent_fail(log=True) def on_remote_closed(self): + print("REMOTE CLOSED") try: self._local_socket.close() except socket.error: pass def start_remote(self, sock): + print("START REMOTE") self._remote_socket = sock def remote_reader(): @@ -87,19 +95,25 @@ def remote_reader(): gevent.spawn(remote_reader) def send_remote(self, data): + print("SEND REMOTE %d" % len(data)) if len(data) == 0: return + self._remote_socket.send(data) def send_local(self, data): + print("SEND LOCAL %d" % len(data)) if len(data) == 0: return + self._local_socket.send(data) def close_local(self): + print("CLOSE LOCAL") self._local_socket.close() def close_remote(self): + print("CLOSE REMOTE") self._remote_socket.close() def _check_for_schema(self): diff --git a/cachebrowser/proxybeta.py b/cachebrowser/proxybeta.py new file mode 100644 index 0000000..7be3f62 --- /dev/null +++ b/cachebrowser/proxybeta.py @@ -0,0 +1,211 @@ +import re + +import errno +import requests +import json +import sys + +import socket + +from cachebrowser import dns +from cachebrowser.common import silent_fail, logger +from cachebrowser.network import HttpConnectionHandler, gevent, ConnectionHandler + +count = 0 + +class ProxyHandler(HttpConnectionHandler): + def on_request(self, env, start_response): + method = env['REQUEST_METHOD'] + + if method == 'CONNECT': + return self.proxy_https(env, start_response) + else: + return self.proxy_http(env, start_response) + + def proxy_http(self, env, start_response): + headers = self.extract_http_headers(env) + body = self.extract_http_body(env) + + url = env['PATH_INFO'] + match = re.match('http://([^/?]+)(?:.+)?', url) + host = match.group(1) + ip, cachebrowsed = dns.resolve_host(host) + changed_url = url.replace(host, ip, 1) + + # TODO Remove PROXY_CONNECTION header + options = { + 'method': env['REQUEST_METHOD'], + 'url': changed_url, + 'headers': headers, + } + + if env['QUERY_STRING']: + options['params'] = env['QUERY_STRING'] + + if body: + options['data'] = body + + session = requests.Session() + request = requests.Request(**options) + prepared_request = request.prepare() + response = session.send(prepared_request, stream=True, allow_redirects=False) + + proxy_logger.log(host, 80, ip, 'http', cachebrowsed=cachebrowsed, url=url) + + start_response("%d %s" % (response.status_code, response.reason), list(response.headers.iteritems())) + + if response.headers.get('Transfer-Encoding', None) == 'chunked': + return response.raw + return response.raw.data + + def extract_http_headers(self, env): + headers = {} + for key in env: + if key.startswith('HTTP_'): + headers[key.replace('HTTP_', '').title()] = env[key] + return headers + + def extract_http_body(self, env): + return env['wsgi.input'].read() + + +class SSLProxyHandler(ConnectionHandler): + def __init__(self, *args, **kwargs): + super(SSLProxyHandler, self).__init__(*args, **kwargs) + self._local_socket = None + self._remote_socket = None + + self.remote_connection_established = False + + self.on_data = self.on_local_data + self.on_close = self.on_local_closed + + def on_connect(self, sock, address): + self._local_socket = sock + # logger.debug("New proxy connection established with %s" % str(self.address)) + + # @silent_fail(log=True) + def on_local_data(self, data): + # print("LOCAL DATA %d" % len(data)) + # print(data) + if len(data) == 0: + return + + if not self.remote_connection_established: + match = re.match("(?:CONNECT) ([^:]+)(?:[:](\d+))? \w+", data) + host = match.group(1) + port = int(match.group(2) or 443) + + ip, cachebrowsed = dns.resolve_host(host) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((ip, port)) + self.remote_connection_established = True + self.start_remote(sock) + + self.send_local("HTTP/1.1 200 OK\r\n\r\n") + + proxy_logger.log(host, port, ip, 'https') + else: + self.send_remote(data) + + # @silent_fail(log=True) + def on_remote_data(self, data): + # print("REMOTE DATA %d" % len(data)) + # print(data) + if len(data) == 0: + return + + self.send_local(data) + + # @silent_fail(log=True) + def on_local_closed(self): + print("LOCAL CLOSED") + if self._remote_socket is None: + return + + try: + self._remote_socket.close() + except socket.error: + pass + + # @silent_fail(log=True) + def on_remote_closed(self): + print("REMOTE CLOSED") + try: + self._local_socket.close() + except socket.error: + pass + + def start_remote(self, sock): + global count + print("START REMOTE") + self._remote_socket = sock + + def remote_reader(): + global count + count += 1 + print(">>>>>>>>>>>>> GOING UP %d" % count) + try: + while True: + buff = sock.recv(1024) + gevent.sleep(0) + if not buff: + self.on_remote_closed() + break + + self.on_remote_data(buff) + except socket.error as e: + if e.errno != errno.EBADF: + logger.error(e) + # TODO REMOVE + raise + except Exception as e: + logger.error(e) + # TODO REMOVE + raise + count -= 1 + print(">>>>>>>>>>>>> GOING DOWN %d" % count) + + + gevent.spawn(remote_reader) + + + def send_remote(self, data): + # print("SEND REMOTE %d" % len(data)) + if len(data) == 0: + return + + self._remote_socket.send(data) + + def send_local(self, data): + # print("SEND LOCAL %d" % len(data)) + if len(data) == 0: + return + + self._local_socket.send(data) + + def close_local(self): + print("CLOSE LOCAL") + self._local_socket.close() + + def close_remote(self): + print("CLOSE REMOTE") + self._remote_socket.close() + + +class ProxyLogger(object): + def __init__(self): + self.handlers = [] + + def log(self, host, port, target, scheme, url=None, cachebrowsed=False): + for handler in self.handlers: + handler(host=host, port=port, target=target, scheme=scheme, url=url, cachebrowsed=cachebrowsed) + + def register_handler(self, handler): + self.handlers.append(handler) + + def remove_handler(self, handler): + self.handlers.remove(handler) + +proxy_logger = ProxyLogger() \ No newline at end of file diff --git a/data/local_bootstrap.yaml b/data/local_bootstrap.yaml index 415cf4f..58f6314 100644 --- a/data/local_bootstrap.yaml +++ b/data/local_bootstrap.yaml @@ -8,4 +8,5 @@ - type: host name: www.cachebrowser.info ssl: true - cdn: cbcdn \ No newline at end of file + cdn: cbcdn +