extmod/asyncio: Add ssl support with SSLContext.

This adds asyncio ssl support with SSLContext and the corresponding
tests in `tests/net_inet` and `tests/multi_net`.

Note that not doing the handshake on connect will delegate the handshake to
the following `mbedtls_ssl_read/write` calls.  However if the handshake
fails when a client certificate is required and not presented by the peer,
it needs to be notified of this handshake error (otherwise it will hang
until timeout if any).  Finally at MicroPython side raise the proper
mbedtls error code and message.
Signed-off-by: Carlos Gil <carlosgilglez@gmail.com>

.gitattributes

@@ -13,6 +13,7 @@
 *.jpg binary
 *.dxf binary
 *.mpy binary
+*.der binary
 
 # These should also not be modified by git.
 tests/basics/string_cr_conversion.py -text

extmod/asyncio/stream.py

@@ -63,6 +63,8 @@ class Stream:
         while True:
             yield core._io_queue.queue_read(self.s)
             l2 = self.s.readline()  # may do multiple reads but won't block
+            if l2 is None:
+                continue
             l += l2
             if not l2 or l[-1] == 10:  # \n (check l in case l2 is str)
                 return l
@@ -100,19 +102,29 @@ StreamWriter = Stream
 # Create a TCP stream connection to a remote host
 #
 # async
-def open_connection(host, port):
+def open_connection(host, port, ssl=None, server_hostname=None):
     from errno import EINPROGRESS
     import socket
 
     ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0]  # TODO this is blocking!
     s = socket.socket(ai[0], ai[1], ai[2])
     s.setblocking(False)
-    ss = Stream(s)
     try:
         s.connect(ai[-1])
     except OSError as er:
         if er.errno != EINPROGRESS:
             raise er
+    # wrap with SSL, if requested
+    if ssl:
+        if ssl is True:
+            import ssl as _ssl
+
+            ssl = _ssl.SSLContext(_ssl.PROTOCOL_TLS_CLIENT)
+        if not server_hostname:
+            server_hostname = host
+        s = ssl.wrap_socket(s, server_hostname=server_hostname, do_handshake_on_connect=False)
+        s.setblocking(False)
+    ss = Stream(s)
     yield core._io_queue.queue_write(s)
     return ss, ss
 
@@ -135,7 +147,7 @@ class Server:
     async def wait_closed(self):
         await self.task
 
-    async def _serve(self, s, cb):
+    async def _serve(self, s, cb, ssl):
         self.state = False
         # Accept incoming connections
         while True:
@@ -156,6 +168,13 @@ class Server:
             except:
                 # Ignore a failed accept
                 continue
+            if ssl:
+                try:
+                    s2 = ssl.wrap_socket(s2, server_side=True, do_handshake_on_connect=False)
+                except OSError as e:
+                    core.sys.print_exception(e)
+                    s2.close()
+                    continue
             s2.setblocking(False)
             s2s = Stream(s2, {"peername": addr})
             core.create_task(cb(s2s, s2s))
@@ -163,7 +182,7 @@ class Server:
 
 # Helper function to start a TCP stream server, running as a new task
 # TODO could use an accept-callback on socket read activity instead of creating a task
-async def start_server(cb, host, port, backlog=5):
+async def start_server(cb, host, port, backlog=5, ssl=None):
     import socket
 
     # Create and bind server socket.
@@ -176,7 +195,7 @@ async def start_server(cb, host, port, backlog=5):
 
     # Create and return server object and task.
     srv = Server()
-    srv.task = core.create_task(srv._serve(s, cb))
+    srv.task = core.create_task(srv._serve(s, cb, ssl))
     try:
         # Ensure that the _serve task has been scheduled so that it gets to
         # handle cancellation.

extmod/modssl_mbedtls.c

@@ -166,6 +166,46 @@ STATIC NORETURN void mbedtls_raise_error(int err) {
     #endif
 }
 
+STATIC void ssl_check_async_handshake_failure(mp_obj_ssl_socket_t *sslsock, int *errcode) {
+    if (
+        #if MBEDTLS_VERSION_NUMBER >= 0x03000000
+        (*errcode < 0) && (mbedtls_ssl_is_handshake_over(&sslsock->ssl) == 0) && (*errcode != MBEDTLS_ERR_SSL_CONN_EOF)
+        #else
+        (*errcode < 0) && (*errcode != MBEDTLS_ERR_SSL_CONN_EOF)
+        #endif
+        ) {
+        // Asynchronous handshake is done by mbdetls_ssl_read/write.  If the return code is
+        // MBEDTLS_ERR_XX (i.e < 0) and the handshake is not done due to a handshake failure,
+        // then notify peer with proper error code and raise local error with mbedtls_raise_error.
+
+        if (*errcode == MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE) {
+            // Check if TLSv1.3 and use proper alert for this case (to be implemented)
+            // uint8_t alert = MBEDTLS_SSL_ALERT_MSG_CERT_REQUIRED; tlsv1.3
+            // uint8_t alert = MBEDTLS_SSL_ALERT_MSG_HANDSHAKE_FAILURE; tlsv1.2
+            mbedtls_ssl_send_alert_message(&sslsock->ssl, MBEDTLS_SSL_ALERT_LEVEL_FATAL,
+                MBEDTLS_SSL_ALERT_MSG_HANDSHAKE_FAILURE);
+        }
+
+        if (*errcode == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED) {
+            // The certificate may have been rejected for several reasons.
+            char xcbuf[256];
+            uint32_t flags = mbedtls_ssl_get_verify_result(&sslsock->ssl);
+            int ret = mbedtls_x509_crt_verify_info(xcbuf, sizeof(xcbuf), "\n", flags);
+            // The length of the string written (not including the terminated nul byte),
+            // or a negative err code.
+            if (ret > 0) {
+                sslsock->sock = MP_OBJ_NULL;
+                mbedtls_ssl_free(&sslsock->ssl);
+                mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("%s"), xcbuf);
+            }
+        }
+
+        sslsock->sock = MP_OBJ_NULL;
+        mbedtls_ssl_free(&sslsock->ssl);
+        mbedtls_raise_error(*errcode);
+    }
+}
+
 /******************************************************************************/
 // SSLContext type.
 
@@ -614,6 +654,7 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
     } else {
         o->last_error = ret;
     }
+    ssl_check_async_handshake_failure(o, &ret);
     *errcode = ret;
     return MP_STREAM_ERROR;
 }
@@ -642,6 +683,7 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in
     } else {
         o->last_error = ret;
     }
+    ssl_check_async_handshake_failure(o, &ret);
     *errcode = ret;
     return MP_STREAM_ERROR;
 }

tests/multi_net/asyncio_tls_server_client.py

+# Test asyncio TCP server and client with TLS, transferring some data.
+
+try:
+    import os
+    import asyncio
+    import ssl
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+PORT = 8000
+
+# These are test certificates. See tests/README.md for details.
+cert = cafile = "multi_net/rsa_cert.der"
+key = "multi_net/rsa_key.der"
+
+try:
+    os.stat(cafile)
+    os.stat(key)
+except OSError:
+    print("SKIP")
+    raise SystemExit
+
+
+async def handle_connection(reader, writer):
+    data = await reader.read(100)
+    print("echo:", data)
+    writer.write(data)
+    await writer.drain()
+
+    print("close")
+    writer.close()
+    await writer.wait_closed()
+
+    print("done")
+    ev.set()
+
+
+async def tcp_server():
+    global ev
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(cert, key)
+    ev = asyncio.Event()
+    server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx)
+    print("server running")
+    multitest.next()
+    async with server:
+        await asyncio.wait_for(ev.wait(), 10)
+
+
+async def tcp_client(message):
+    client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    client_ctx.verify_mode = ssl.CERT_REQUIRED
+    client_ctx.load_verify_locations(cafile=cafile)
+    reader, writer = await asyncio.open_connection(
+        IP, PORT, ssl=client_ctx, server_hostname="micropython.local"
+    )
+    print("write:", message)
+    writer.write(message)
+    await writer.drain()
+    data = await reader.read(100)
+    print("read:", data)
+
+
+def instance0():
+    multitest.globals(IP=multitest.get_network_ip())
+    asyncio.run(tcp_server())
+
+
+def instance1():
+    multitest.next()
+    asyncio.run(tcp_client(b"client data"))

tests/multi_net/asyncio_tls_server_client.py.exp

+--- instance0 ---
+server running
+echo: b'client data'
+close
+done
+--- instance1 ---
+write: b'client data'
+read: b'client data'

tests/multi_net/asyncio_tls_server_client_cert_required_error.py

+# Test asyncio TCP server and client with TLS, giving a cert required error.
+
+try:
+    import os
+    import asyncio
+    import ssl
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+PORT = 8000
+
+# These are test certificates. See tests/README.md for details.
+cert = cafile = "multi_net/rsa_cert.der"
+key = "multi_net/rsa_key.der"
+
+try:
+    os.stat(cafile)
+    os.stat(key)
+except OSError:
+    print("SKIP")
+    raise SystemExit
+
+
+async def handle_connection(reader, writer):
+    print("handle connection")
+    try:
+        data = await reader.read(100)
+    except Exception as e:
+        print(e)
+    ev.set()
+
+
+async def tcp_server():
+    global ev
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(cert, key)
+    server_ctx.verify_mode = ssl.CERT_REQUIRED
+    server_ctx.load_verify_locations(cafile=cert)
+    ev = asyncio.Event()
+    server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx)
+    print("server running")
+    multitest.next()
+    async with server:
+        await asyncio.wait_for(ev.wait(), 10)
+        multitest.wait("finished")
+        print("server done")
+
+
+async def tcp_client(message):
+    client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    client_ctx.verify_mode = ssl.CERT_REQUIRED
+    client_ctx.load_verify_locations(cafile=cafile)
+    reader, writer = await asyncio.open_connection(
+        IP, PORT, ssl=client_ctx, server_hostname="micropython.local"
+    )
+    try:
+        print("write:", message)
+        writer.write(message)
+        print("drain")
+        await writer.drain()
+    except Exception as e:
+        print(e)
+    print("client done")
+    multitest.broadcast("finished")
+
+
+def instance0():
+    multitest.globals(IP=multitest.get_network_ip())
+    asyncio.run(tcp_server())
+
+
+def instance1():
+    multitest.next()
+    asyncio.run(tcp_client(b"client data"))

tests/multi_net/asyncio_tls_server_client_cert_required_error.py.exp

+--- instance0 ---
+server running
+handle connection
+(-29824, 'MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE')
+server done
+--- instance1 ---
+write: b'client data'
+drain
+(-30592, 'MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE')
+client done

tests/multi_net/asyncio_tls_server_client_readline.py

+# Test asyncio TCP server and client with TLS, using readline() to read data.
+
+try:
+    import os
+    import asyncio
+    import ssl
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+PORT = 8000
+
+# These are test certificates. See tests/README.md for details.
+cert = cafile = "multi_net/rsa_cert.der"
+key = "multi_net/rsa_key.der"
+
+try:
+    os.stat(cafile)
+    os.stat(key)
+except OSError:
+    print("SKIP")
+    raise SystemExit
+
+
+async def handle_connection(reader, writer):
+    data = await reader.readline()
+    print("echo:", data)
+    data2 = await reader.readline()
+    print("echo:", data2)
+    writer.write(data + data2)
+    await writer.drain()
+
+    print("close")
+    writer.close()
+    await writer.wait_closed()
+
+    print("done")
+    ev.set()
+
+
+async def tcp_server():
+    global ev
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(cert, key)
+    ev = asyncio.Event()
+    server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx)
+    print("server running")
+    multitest.next()
+    async with server:
+        await asyncio.wait_for(ev.wait(), 10)
+
+
+async def tcp_client(message):
+    client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    client_ctx.verify_mode = ssl.CERT_REQUIRED
+    client_ctx.load_verify_locations(cafile=cafile)
+    reader, writer = await asyncio.open_connection(
+        IP, PORT, ssl=client_ctx, server_hostname="micropython.local"
+    )
+    print("write:", message)
+    writer.write(message)
+    await writer.drain()
+    data = await reader.readline()
+    print("read:", data)
+    data2 = await reader.readline()
+    print("read:", data2)
+
+
+def instance0():
+    multitest.globals(IP=multitest.get_network_ip())
+    asyncio.run(tcp_server())
+
+
+def instance1():
+    multitest.next()
+    asyncio.run(tcp_client(b"client data\nclient data2\n"))

tests/multi_net/asyncio_tls_server_client_readline.py.exp

+--- instance0 ---
+server running
+echo: b'client data\n'
+echo: b'client data2\n'
+close
+done
+--- instance1 ---
+write: b'client data\nclient data2\n'
+read: b'client data\n'
+read: b'client data2\n'

tests/multi_net/asyncio_tls_server_client_verify_error.py

+# Test asyncio TCP server and client with TLS, and an incorrect server_hostname.
+
+try:
+    import os
+    import asyncio
+    import ssl
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+PORT = 8000
+
+# These are test certificates. See tests/README.md for details.
+cert = cafile = "multi_net/rsa_cert.der"
+key = "multi_net/rsa_key.der"
+
+try:
+    os.stat(cafile)
+    os.stat(key)
+except OSError:
+    print("SKIP")
+    raise SystemExit
+
+
+async def handle_connection(reader, writer):
+    print("handle connection")
+    try:
+        data = await reader.read(100)
+    except Exception as e:
+        print(e)
+    ev.set()
+
+
+async def tcp_server():
+    global ev
+
+    server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server_ctx.load_cert_chain(cert, key)
+    ev = asyncio.Event()
+    server = await asyncio.start_server(handle_connection, "0.0.0.0", PORT, ssl=server_ctx)
+    print("server running")
+    multitest.next()
+    async with server:
+        await asyncio.wait_for(ev.wait(), 10)
+    print("server done")
+    multitest.broadcast("finished")
+
+
+async def tcp_client(message):
+    client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    client_ctx.verify_mode = ssl.CERT_REQUIRED
+    client_ctx.load_verify_locations(cafile=cafile)
+    reader, writer = await asyncio.open_connection(
+        IP,
+        PORT,
+        ssl=client_ctx,
+        server_hostname="foobar.local",  # incorrect server_hostname
+    )
+    try:
+        print("write:", message)
+        writer.write(message)
+        print("drain")
+        await writer.drain()
+    except Exception as e:
+        print(e)
+    multitest.wait("finished")
+    print("client done")
+
+
+def instance0():
+    multitest.globals(IP=multitest.get_network_ip())
+    asyncio.run(tcp_server())
+
+
+def instance1():
+    multitest.next()
+    asyncio.run(tcp_client(b"client data"))

tests/multi_net/asyncio_tls_server_client_verify_error.py.exp

+--- instance0 ---
+server running
+handle connection
+(-30592, 'MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE')
+server done
+--- instance1 ---
+write: b'client data'
+drain
+
+The certificate Common Name (CN) does not match with the expected CN
+
+client done

tests/net_inet/asyncio_tls_open_connection_readline.py

+import ssl
+import os
+import asyncio
+
+# This certificate was obtained from micropython.org using openssl:
+# $ openssl s_client -showcerts -connect micropython.org:443 </dev/null 2>/dev/null
+# The certificate is from Let's Encrypt:
+# 1 s:/C=US/O=Let's Encrypt/CN=R3
+#   i:/C=US/O=Internet Security Research Group/CN=ISRG Root X1
+# Validity
+#            Not Before: Sep  4 00:00:00 2020 GMT
+#            Not After : Sep 15 16:00:00 2025 GMT
+# Copy PEM content to a file (certmpy.pem) and convert to DER e.g.
+# $ openssl x509 -in certmpy.pem -out certmpy.der -outform DER
+# Then convert to hex format, eg using binascii.hexlify(data).
+
+# Note that the instructions above is to obtain an intermediate
+# root CA cert that works for MicroPython. However CPython needs the ultimate root CA
+# cert from ISRG
+
+ca_cert_chain = "isrg.der"
+
+try:
+    os.stat(ca_cert_chain)
+except OSError:
+    print("SKIP")
+    raise SystemExit
+
+with open(ca_cert_chain, "rb") as ca:
+    cadata = ca.read()
+
+client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+client_ctx.verify_mode = ssl.CERT_REQUIRED
+client_ctx.load_verify_locations(cadata=cadata)
+
+
+async def http_get(url, port, sslctx):
+    reader, writer = await asyncio.open_connection(url, port, ssl=sslctx)
+
+    print("write GET")
+    writer.write(b"GET / HTTP/1.0\r\n\r\n")
+    await writer.drain()
+
+    print("read response")
+    while True:
+        data = await reader.readline()
+        # avoid printing datetime which makes the test fail
+        if b"GMT" not in data:
+            print("read:", data)
+        if not data:
+            break
+
+    print("close")
+    writer.close()
+    await writer.wait_closed()
+    print("done")
+
+
+asyncio.run(http_get("micropython.org", 443, client_ctx))