From 13e0c451c147ee3c541083469b429cf93bf9cdcb Mon Sep 17 00:00:00 2001 From: Ari Braginsky Date: Tue, 9 Jul 2019 14:47:38 -0700 Subject: [PATCH 1/4] - Fixes --- adafruit_esp32spi/adafruit_esp32spi.py | 0 .../adafruit_esp32spi_requests.py | 22 +++++++++++++++++++ adafruit_esp32spi/adafruit_esp32spi_socket.py | 1 + 3 files changed, 23 insertions(+) mode change 100644 => 100755 adafruit_esp32spi/adafruit_esp32spi.py mode change 100644 => 100755 adafruit_esp32spi/adafruit_esp32spi_socket.py diff --git a/adafruit_esp32spi/adafruit_esp32spi.py b/adafruit_esp32spi/adafruit_esp32spi.py old mode 100644 new mode 100755 diff --git a/adafruit_esp32spi/adafruit_esp32spi_requests.py b/adafruit_esp32spi/adafruit_esp32spi_requests.py index 3857061..ebdb3de 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_requests.py +++ b/adafruit_esp32spi/adafruit_esp32spi_requests.py @@ -159,9 +159,31 @@ def request(method, url, data=None, json=None, headers=None, stream=False, timeo host, port = host.split(":", 1) port = int(port) + print("host: ", host) + print("port: ", port) + addr_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0] + print("addr_info: ", addr_info) + sock = socket.socket(addr_info[0], addr_info[1], addr_info[2]) + print("socket: ", socket) + resp = Response(sock) # our response + # print("response: ", resp) + +# self.socket = sock +# self.encoding = "utf-8" +# self._cached = None +# self.status_code = None +# self.reason = None +# self._read_so_far = 0 +# self.headers = {} + + # if resp: + # print("response headers: ", resp.headers) + # print("response reason: ", resp.reason) + # print("response status code: ", resp.status_code) + sock.settimeout(timeout) # socket read timeout diff --git a/adafruit_esp32spi/adafruit_esp32spi_socket.py b/adafruit_esp32spi/adafruit_esp32spi_socket.py old mode 100644 new mode 100755 index ccf5b4f..4c67c79 --- a/adafruit_esp32spi/adafruit_esp32spi_socket.py +++ b/adafruit_esp32spi/adafruit_esp32spi_socket.py @@ -88,6 +88,7 @@ def readline(self): """Attempt to return as many bytes as we can up to but not including '\r\n'""" #print("Socket readline") stamp = time.monotonic() + # print("buffer: ", self._buffer) while b'\r\n' not in self._buffer: # there's no line already in there, read some more avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) From cf1d70d716ee0b5e0ad9d72acb701c260176baa1 Mon Sep 17 00:00:00 2001 From: Ari Braginsky Date: Wed, 17 Jul 2019 18:37:17 -0700 Subject: [PATCH 2/4] - Updates --- adafruit_esp32spi/adafruit_esp32spi.py | 12 +++-- .../adafruit_esp32spi_requests.py | 10 +++- adafruit_esp32spi/adafruit_esp32spi_socket.py | 47 +++++++++++++++++-- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/adafruit_esp32spi/adafruit_esp32spi.py b/adafruit_esp32spi/adafruit_esp32spi.py index e5f109a..e8b8224 100755 --- a/adafruit_esp32spi/adafruit_esp32spi.py +++ b/adafruit_esp32spi/adafruit_esp32spi.py @@ -251,7 +251,10 @@ def _read_bytes(self, spi, buffer, start=0, end=None): def _wait_spi_char(self, spi, desired): """Read a byte with a time-out, and if we get it, check that its what we expect""" times = time.monotonic() - while (time.monotonic() - times) < 0.1: + # while (time.monotonic() - times) < 0.1: + + # local edit 20190717 + while (time.monotonic() - times) < 1: r = self._read_byte(spi) if r == _ERR_CMD: raise RuntimeError("Error response to command") @@ -577,8 +580,11 @@ def socket_available(self, socket_num): self._socknum_ll[0][0] = socket_num resp = self._send_command_get_response(_AVAIL_DATA_TCP_CMD, self._socknum_ll) reply = struct.unpack(' 2: reason = line[2].rstrip() + + print("status: ", status) + print("reason: ", reason) + while True: line = sock.readline() if not line or line == b"\r\n": + print("we're done with request") break #print("**line: ", line) diff --git a/adafruit_esp32spi/adafruit_esp32spi_socket.py b/adafruit_esp32spi/adafruit_esp32spi_socket.py index 4c67c79..efa1227 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_socket.py +++ b/adafruit_esp32spi/adafruit_esp32spi_socket.py @@ -67,11 +67,13 @@ def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None): self._buffer = b'' self._socknum = _the_interface.get_socket() self.settimeout(0) + print("socket created") def connect(self, address, conntype=None): """Connect the socket to the 'address' (which can be 32bit packed IP or a hostname string). 'conntype' is an extra that may indicate SSL or not, depending on the underlying interface""" + print("connecting...") host, port = address if conntype is None: conntype = _the_interface.TCP_MODE @@ -86,12 +88,14 @@ def write(self, data): # pylint: disable=no-self-use def readline(self): """Attempt to return as many bytes as we can up to but not including '\r\n'""" - #print("Socket readline") + # print("Socket readline") stamp = time.monotonic() # print("buffer: ", self._buffer) while b'\r\n' not in self._buffer: # there's no line already in there, read some more avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) + if avail != 0: + print("avail: ", avail) if avail: self._buffer += _the_interface.socket_read(self._socknum, avail) elif self._timeout > 0 and time.monotonic() - stamp > self._timeout: @@ -104,7 +108,7 @@ def readline(self): def read(self, size=0): """Read up to 'size' bytes from the socket, this may be buffered internally! If 'size' isnt specified, return everything in the buffer.""" - #print("Socket read", size) + # print("Socket read: ", size) if size == 0: # read as much as we can at the moment while True: avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) @@ -120,21 +124,49 @@ def read(self, size=0): stamp = time.monotonic() to_read = size - len(self._buffer) + + # print("size: ", size) + # print("buffer length: ", len(self._buffer)) + + # if this is too short, could end too early and size of file written will not match + # the content-length from server... + # + # read_timeout = 1 + # read_timeout = self._timeout + read_timeout = 8 + received = [] while to_read > 0: - #print("Bytes to read:", to_read) - avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) + # print("Bytes to read:", to_read) + available_bytes = _the_interface.socket_available(self._socknum) + + # if available_bytes > 0: + # print("available bytes on sock: ", available_bytes) + + # if available_bytes > MAX_PACKET: + # print("Warning: available bytes is > MAX_PACKET: ", MAX_PACKET) + + avail = min(available_bytes, MAX_PACKET) + if avail: + # print("avail: ", avail) stamp = time.monotonic() recv = _the_interface.socket_read(self._socknum, min(to_read, avail)) received.append(recv) to_read -= len(recv) gc.collect() - if self._timeout > 0 and time.monotonic() - stamp > self._timeout: + # else: + # print("nothing left to read! waiting to timeout... ", (time.monotonic() - stamp)) + #if self._timeout > 0 and time.monotonic() - stamp > self._timeout: + if read_timeout > 0 and time.monotonic() - stamp > read_timeout: + #print("socket.read timeout ", self._timeout) + print("socket.read timeout ", read_timeout) break #print(received) self._buffer += b''.join(received) + # print("len(self._buffer) ", len(self._buffer)) + ret = None if len(self._buffer) == size: ret = self._buffer @@ -142,11 +174,16 @@ def read(self, size=0): else: ret = self._buffer[:size] self._buffer = self._buffer[size:] + + # print("size: ", size) + # print("len(ret) ", len(ret)) + gc.collect() return ret def settimeout(self, value): """Set the read timeout for sockets, if value is 0 it will block""" + print("setting timeout: ", value) self._timeout = value def close(self): From 9ff13aa4d494c88ef53ab56f1e09a9feb800b4c6 Mon Sep 17 00:00:00 2001 From: Ari Braginsky Date: Thu, 18 Jul 2019 13:47:12 -0700 Subject: [PATCH 3/4] - Fixes --- adafruit_esp32spi/adafruit_esp32spi_requests.py | 15 ++++++++++----- adafruit_esp32spi/adafruit_esp32spi_socket.py | 9 ++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/adafruit_esp32spi/adafruit_esp32spi_requests.py b/adafruit_esp32spi/adafruit_esp32spi_requests.py index 8217053..44ce871 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_requests.py +++ b/adafruit_esp32spi/adafruit_esp32spi_requests.py @@ -122,13 +122,18 @@ def iter_content(self, chunk_size=1, decode_unicode=False): raise NotImplementedError("Unicode not supported") while True: - chunk = self.socket.read(chunk_size) - if chunk: - yield chunk - else: - print("NO CHUNK") + try: + chunk = self.socket.read(chunk_size) + if chunk: + yield chunk + else: + print("NO CHUNKS LEFT") + return + except MemoryError as e: + print("MemoryError reading from socket! ", e) return + # pylint: disable=too-many-branches, too-many-statements, unused-argument, too-many-arguments, too-many-locals def request(method, url, data=None, json=None, headers=None, stream=False, timeout=1): """Perform an HTTP request to the given url which we will parse to determine diff --git a/adafruit_esp32spi/adafruit_esp32spi_socket.py b/adafruit_esp32spi/adafruit_esp32spi_socket.py index efa1227..8af35d9 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_socket.py +++ b/adafruit_esp32spi/adafruit_esp32spi_socket.py @@ -113,7 +113,14 @@ def read(self, size=0): while True: avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) if avail: + + # BUG:20190718 memory allocation failed here sometimes due to + # internal buffering (see function description) self._buffer += _the_interface.socket_read(self._socknum, avail) + + # 20190718 + gc.collect() + else: break gc.collect() @@ -133,7 +140,7 @@ def read(self, size=0): # # read_timeout = 1 # read_timeout = self._timeout - read_timeout = 8 + read_timeout = 15 received = [] while to_read > 0: From 85248eff0727172ef3c656fb916f670d6a2264ea Mon Sep 17 00:00:00 2001 From: Ari Braginsky Date: Thu, 18 Jul 2019 14:43:14 -0700 Subject: [PATCH 4/4] - Updates --- .../adafruit_esp32spi_requests.py | 10 +++- adafruit_esp32spi/adafruit_esp32spi_socket.py | 50 ++++++++++++------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/adafruit_esp32spi/adafruit_esp32spi_requests.py b/adafruit_esp32spi/adafruit_esp32spi_requests.py index 44ce871..ca21d2d 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_requests.py +++ b/adafruit_esp32spi/adafruit_esp32spi_requests.py @@ -225,7 +225,15 @@ def request(method, url, data=None, json=None, headers=None, stream=False, timeo if data: sock.write(bytes(data, 'utf-8')) - line = sock.readline() + try: + line = sock.readline() + except MemoryError as e: + print("MemoryError while reading line from socket for initial request! ", e) + sock.close() + resp.status_code = "500" + resp.reason = "MemoryError" + return resp + print("line: ", line) line = line.split(None, 2) status = int(line[1]) diff --git a/adafruit_esp32spi/adafruit_esp32spi_socket.py b/adafruit_esp32spi/adafruit_esp32spi_socket.py index 8af35d9..ee3f62d 100755 --- a/adafruit_esp32spi/adafruit_esp32spi_socket.py +++ b/adafruit_esp32spi/adafruit_esp32spi_socket.py @@ -93,14 +93,22 @@ def readline(self): # print("buffer: ", self._buffer) while b'\r\n' not in self._buffer: # there's no line already in there, read some more - avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) - if avail != 0: - print("avail: ", avail) - if avail: - self._buffer += _the_interface.socket_read(self._socknum, avail) - elif self._timeout > 0 and time.monotonic() - stamp > self._timeout: - self.close() # Make sure to close socket so that we don't exhaust sockets. - raise RuntimeError("Didn't receive full response, failing out") + try: + # BUG:20190718 memory allocation failed here sometimes due to + # internal buffering (see function description) + avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) + if avail != 0: + print("avail: ", avail) + if avail: + self._buffer += _the_interface.socket_read(self._socknum, avail) + # 20190718 + gc.collect() + elif self._timeout > 0 and time.monotonic() - stamp > self._timeout: + self.close() # Make sure to close socket so that we don't exhaust sockets. + raise RuntimeError("Didn't receive full response, failing out") + except MemoryError as e: + print("MemoryError while reading from socket (readline)") + break firstline, self._buffer = self._buffer.split(b'\r\n', 1) gc.collect() return firstline @@ -111,18 +119,22 @@ def read(self, size=0): # print("Socket read: ", size) if size == 0: # read as much as we can at the moment while True: - avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) - if avail: - - # BUG:20190718 memory allocation failed here sometimes due to - # internal buffering (see function description) - self._buffer += _the_interface.socket_read(self._socknum, avail) - - # 20190718 - gc.collect() - - else: + try: + avail = min(_the_interface.socket_available(self._socknum), MAX_PACKET) + if avail: + + # BUG:20190718 memory allocation failed here sometimes due to + # internal buffering (see function description) + self._buffer += _the_interface.socket_read(self._socknum, avail) + + # 20190718 + gc.collect() + else: + break + except MemoryError as e: + print("MemoryError while reading from socket (read)") break + gc.collect() ret = self._buffer self._buffer = b''