py/persistentcode: Remove remaining native qstr linking support.

Support for architecture-specific qstr linking was removed in
d4d53e9e, where native code was changed to
access qstr values via qstr_table.  The only remaining use for the special
qstr link table in persistentcode.c is to support native module written in
C, linked via mpy_ld.py.  But native modules can also use the standard
module-level qstr_table (and obj_table) which was introduced in the .mpy
file reworking in f2040bfc.

This commit removes the remaining native qstr liking support in
persistentcode.c's load_raw_code function, and adds two new relocation
options for constants.qstr_table and constants.obj_table.  mpy_ld.py is
updated to use these relocations options instead of the native qstr link
table.
Signed-off-by: Damien George <damien@micropython.org>

py/persistentcode.c

@@ -71,6 +71,7 @@ STATIC size_t read_uint(mp_reader_t *reader);
 
 typedef struct _reloc_info_t {
     mp_reader_t *reader;
+    mp_module_context_t *context;
     uint8_t *rodata;
     uint8_t *bss;
 } reloc_info_t;
@@ -112,11 +113,17 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) {
                 dest = (uintptr_t)ri->bss;
             }
         } else if (op == 6) {
+            // Destination is qstr_table
+            dest = (uintptr_t)ri->context->constants.qstr_table;
+        } else if (op == 7) {
+            // Destination is obj_table
+            dest = (uintptr_t)ri->context->constants.obj_table;
+        } else if (op == 8) {
             // Destination is mp_fun_table itself
             dest = (uintptr_t)&mp_fun_table;
         } else {
             // Destination is an entry in mp_fun_table
-            dest = ((uintptr_t *)&mp_fun_table)[op - 7];
+            dest = ((uintptr_t *)&mp_fun_table)[op - 9];
         }
         while (n--) {
             *addr_to_adjust++ += dest;
@@ -205,7 +212,7 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
     }
 }
 
-STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
+STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *context) {
     // Load function kind and data length
     size_t kind_len = read_uint(reader);
     int kind = (kind_len & 3) + MP_CODE_BYTECODE;
@@ -239,24 +246,6 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
         MP_PLAT_ALLOC_EXEC(fun_data_len, (void **)&fun_data, &fun_alloc);
         read_bytes(reader, fun_data, fun_data_len);
 
-        if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) {
-            // Parse qstr link table and link native code
-            size_t n_qstr_link = read_uint(reader);
-            for (size_t i = 0; i < n_qstr_link; ++i) {
-                size_t off = read_uint(reader);
-                qstr qst = load_qstr(reader);
-                uint8_t *dest = fun_data + (off >> 2);
-                if ((off & 3) == 0) {
-                    // Generic 16-bit link
-                    dest[0] = qst & 0xff;
-                    dest[1] = (qst >> 8) & 0xff;
-                } else if ((off & 3) == 3) {
-                    // Generic, aligned qstr-object link
-                    *(mp_obj_t *)dest = MP_OBJ_NEW_QSTR(qst);
-                }
-            }
-        }
-
         if (kind == MP_CODE_NATIVE_PY) {
             // Read prelude offset within fun_data, and extract scope flags.
             prelude_offset = read_uint(reader);
@@ -315,7 +304,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
         n_children = read_uint(reader);
         children = m_new(mp_raw_code_t *, n_children + (kind == MP_CODE_NATIVE_PY));
         for (size_t i = 0; i < n_children; ++i) {
-            children[i] = load_raw_code(reader);
+            children[i] = load_raw_code(reader, context);
         }
     }
 
@@ -349,7 +338,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
         #endif
 
         // Relocate and commit code to executable address space
-        reloc_info_t ri = {reader, rodata, bss};
+        reloc_info_t ri = {reader, context, rodata, bss};
         #if defined(MP_PLAT_COMMIT_EXEC)
         void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL;
         fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri);
@@ -429,7 +418,7 @@ mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *
 
     // Load top-level module.
     mp_compiled_module_t cm2;
-    cm2.rc = load_raw_code(reader);
+    cm2.rc = load_raw_code(reader, context);
     cm2.context = context;
 
     #if MICROPY_PERSISTENT_CODE_SAVE
@@ -567,11 +556,6 @@ STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) {
     mp_print_bytes(print, rc->fun_data, rc->fun_data_len);
 
     #if MICROPY_EMIT_MACHINE_CODE
-    if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) {
-        // Save qstr link table for native code
-        mp_print_uint(print, 0);
-    }
-
     if (rc->kind == MP_CODE_NATIVE_PY) {
         // Save prelude size
         mp_print_uint(print, rc->prelude_offset);

tests/micropython/import_mpy_native.py

@@ -74,9 +74,7 @@ user_files = {
         b'\x02' # 2 children
 
             b'\x42' # 8 bytes, no children, viper code
-                b'\x00\x00\x00\x00\x00\x00' # dummy machine code
-                b'\x00\x00' # slot for qstr0
-                b'\x01\x0c\x0aprint\x00' # n_qstr=1, qstr0
+                b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code
                 b'\x00' # scope_flags
 
             b'\x43' # 8 bytes, no children, asm code
@@ -102,7 +100,6 @@ user_files = {
 
             b'\x22' # 4 bytes, no children, viper code
                 b'\x00\x00\x00\x00' # dummy machine code
-                b'\x00' # n_qstr=0
                 b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC
                 b'\x06\x04' # rodata=6 bytes, bss=4 bytes
                 b'rodata' # rodata content

tests/micropython/import_mpy_native_gc.py

@@ -49,9 +49,9 @@ class UserFS:
 # by the required value of sys.implementation._mpy.
 features0_file_contents = {
     # -march=x64
-    0x806: b'M\x06\b\x1f\x01\x004build/features0.native.mpy\x00\x8aB\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x85\x00\x12factorial\x00\x10\r$\x01&\x9f \x01"\xff',
+    0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
     # -march=armv6m
-    0x1006: b'M\x06\x10\x1f\x01\x004build/features0.native.mpy\x00\x88"\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyD\x18\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial\x00\x10\r<\x01>\x9f8\x01:\xff',
+    0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
 }
 
 # Populate armv7m-derived archs based on armv6m.

tools/mpy-tool.py

@@ -976,7 +976,6 @@ class RawCodeNative(RawCode):
         kind,
         fun_data,
         prelude_offset,
-        qstr_links,
         scope_flags,
         n_pos_args,
         type_sig,
@@ -989,7 +988,6 @@ class RawCodeNative(RawCode):
             self.scope_flags = scope_flags
             self.n_pos_args = n_pos_args
 
-        self.qstr_links = qstr_links
         self.type_sig = type_sig
         if config.native_arch in (
             MP_NATIVE_ARCH_X86,
@@ -1196,15 +1194,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments):
         rc = RawCodeBytecode(cm_escaped_name, qstr_table, obj_table, fun_data)
     else:
         # Create native raw code.
-        qstr_links = []
-        if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER):
-            # Read qstr link table.
-            n_qstr_link = reader.read_uint()
-            for _ in range(n_qstr_link):
-                off = reader.read_uint()
-                qst = read_qstr(reader, segments)
-                qstr_links.append((off >> 2, off & 3, qst))
-
         native_scope_flags = 0
         native_n_pos_args = 0
         native_type_sig = 0
@@ -1242,7 +1231,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments):
             kind,
             fun_data,
             prelude_offset,
-            qstr_links,
             native_scope_flags,
             native_n_pos_args,
             native_type_sig,

tools/mpy_ld.py

@@ -36,6 +36,8 @@ import makeqstrdata as qstrutil
 
 # MicroPython constants
 MPY_VERSION = 6
+MP_CODE_BYTECODE = 2
+MP_CODE_NATIVE_VIPER = 4
 MP_NATIVE_ARCH_X86 = 1
 MP_NATIVE_ARCH_X64 = 2
 MP_NATIVE_ARCH_ARMV6M = 4
@@ -44,8 +46,7 @@ MP_NATIVE_ARCH_ARMV7EMSP = 7
 MP_NATIVE_ARCH_ARMV7EMDP = 8
 MP_NATIVE_ARCH_XTENSA = 9
 MP_NATIVE_ARCH_XTENSAWIN = 10
-MP_CODE_BYTECODE = 2
-MP_CODE_NATIVE_VIPER = 4
+MP_PERSISTENT_OBJ_STR = 5
 MP_SCOPE_FLAG_VIPERRELOC = 0x10
 MP_SCOPE_FLAG_VIPERRODATA = 0x20
 MP_SCOPE_FLAG_VIPERBSS = 0x40
@@ -110,21 +111,20 @@ def asm_jump_xtensa(entry):
 
 
 class ArchData:
-    def __init__(self, name, mpy_feature, qstr_entry_size, word_size, arch_got, asm_jump):
+    def __init__(self, name, mpy_feature, word_size, arch_got, asm_jump, *, separate_rodata=False):
         self.name = name
         self.mpy_feature = mpy_feature
-        self.qstr_entry_size = qstr_entry_size
+        self.qstr_entry_size = 2
         self.word_size = word_size
         self.arch_got = arch_got
         self.asm_jump = asm_jump
-        self.separate_rodata = name == "EM_XTENSA" and qstr_entry_size == 4
+        self.separate_rodata = separate_rodata
 
 
 ARCH_DATA = {
     "x86": ArchData(
         "EM_386",
         MP_NATIVE_ARCH_X86 << 2,
-        2,
         4,
         (R_386_PC32, R_386_GOT32, R_386_GOT32X),
         asm_jump_x86,
@@ -132,7 +132,6 @@ ARCH_DATA = {
     "x64": ArchData(
         "EM_X86_64",
         MP_NATIVE_ARCH_X64 << 2,
-        2,
         8,
         (R_X86_64_GOTPCREL, R_X86_64_REX_GOTPCRELX),
         asm_jump_x86,
@@ -140,7 +139,6 @@ ARCH_DATA = {
     "armv6m": ArchData(
         "EM_ARM",
         MP_NATIVE_ARCH_ARMV6M << 2,
-        2,
         4,
         (R_ARM_GOT_BREL,),
         asm_jump_thumb,
@@ -148,7 +146,6 @@ ARCH_DATA = {
     "armv7m": ArchData(
         "EM_ARM",
         MP_NATIVE_ARCH_ARMV7M << 2,
-        2,
         4,
         (R_ARM_GOT_BREL,),
         asm_jump_thumb2,
@@ -156,7 +153,6 @@ ARCH_DATA = {
     "armv7emsp": ArchData(
         "EM_ARM",
         MP_NATIVE_ARCH_ARMV7EMSP << 2,
-        2,
         4,
         (R_ARM_GOT_BREL,),
         asm_jump_thumb2,
@@ -164,7 +160,6 @@ ARCH_DATA = {
     "armv7emdp": ArchData(
         "EM_ARM",
         MP_NATIVE_ARCH_ARMV7EMDP << 2,
-        2,
         4,
         (R_ARM_GOT_BREL,),
         asm_jump_thumb2,
@@ -172,7 +167,6 @@ ARCH_DATA = {
     "xtensa": ArchData(
         "EM_XTENSA",
         MP_NATIVE_ARCH_XTENSA << 2,
-        2,
         4,
         (R_XTENSA_32, R_XTENSA_PLT),
         asm_jump_xtensa,
@@ -181,9 +175,9 @@ ARCH_DATA = {
         "EM_XTENSA",
         MP_NATIVE_ARCH_XTENSAWIN << 2,
         4,
-        4,
         (R_XTENSA_32, R_XTENSA_PLT),
         asm_jump_xtensa,
+        separate_rodata=True,
     ),
 }
 
@@ -435,8 +429,8 @@ def populate_got(env):
 
     # Create a relocation for each GOT entry
     for got_entry in got_list:
-        if got_entry.name == "mp_fun_table":
-            dest = "mp_fun_table"
+        if got_entry.name in ("mp_native_qstr_table", "mp_native_obj_table", "mp_fun_table"):
+            dest = got_entry.name
         elif got_entry.name.startswith("mp_fun_table+0x"):
             dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size
         elif got_entry.sec_name.startswith(".text"):
@@ -754,19 +748,19 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len):
         env.lit_section = Section("LIT", bytearray(lit_size), env.arch.word_size)
         env.sections.insert(1, env.lit_section)
 
-    # Create section to contain mp_native_qstr_val_table
-    env.qstr_val_section = Section(
-        ".text.QSTR_VAL",
+    # Create section to contain mp_native_qstr_table
+    env.qstr_table_section = Section(
+        ".external.qstr_table",
         bytearray(native_qstr_vals_len * env.arch.qstr_entry_size),
         env.arch.qstr_entry_size,
     )
-    env.sections.append(env.qstr_val_section)
 
-    # Create section to contain mp_native_qstr_obj_table
-    env.qstr_obj_section = Section(
-        ".text.QSTR_OBJ", bytearray(native_qstr_objs_len * env.arch.word_size), env.arch.word_size
+    # Create section to contain mp_native_obj_table
+    env.obj_table_section = Section(
+        ".external.obj_table",
+        bytearray(native_qstr_objs_len * env.arch.word_size),
+        env.arch.word_size,
     )
-    env.sections.append(env.qstr_obj_section)
 
     # Resolve unknown symbols
     mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0)
@@ -796,10 +790,10 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len):
             pass
         elif sym.name == "mp_fun_table":
             sym.section = Section(".external", b"", 0)
-        elif sym.name == "mp_native_qstr_val_table":
-            sym.section = env.qstr_val_section
-        elif sym.name == "mp_native_qstr_obj_table":
-            sym.section = env.qstr_obj_section
+        elif sym.name == "mp_native_qstr_table":
+            sym.section = env.qstr_table_section
+        elif sym.name == "mp_native_obj_table":
+            sym.section = env.obj_table_section
         elif sym.name in env.known_syms:
             sym.resolved = env.known_syms[sym.name]
         else:
@@ -923,13 +917,21 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
     out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
 
     # MPY: n_qstr
-    out.write_uint(1)
+    out.write_uint(1 + len(native_qstr_vals))
 
     # MPY: n_obj
-    out.write_uint(0)
+    out.write_uint(len(native_qstr_objs))
 
     # MPY: qstr table
     out.write_qstr(fmpy)  # filename
+    for q in native_qstr_vals:
+        out.write_qstr(q)
+
+    # MPY: object table
+    for q in native_qstr_objs:
+        out.write_bytes(bytearray([MP_PERSISTENT_OBJ_STR]))
+        out.write_uint(len(q))
+        out.write_bytes(bytes(q, "utf8") + b"\x00")
 
     # MPY: kind/len
     out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE))
@@ -937,17 +939,6 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
     # MPY: machine code
     out.write_bytes(env.full_text)
 
-    # MPY: n_qstr_link (assumes little endian)
-    out.write_uint(len(native_qstr_vals) + len(native_qstr_objs))
-    for q in range(len(native_qstr_vals)):
-        off = env.qstr_val_section.addr + q * env.arch.qstr_entry_size
-        out.write_uint(off << 2)
-        out.write_qstr(native_qstr_vals[q])
-    for q in range(len(native_qstr_objs)):
-        off = env.qstr_obj_section.addr + q * env.arch.word_size
-        out.write_uint(off << 2 | 3)
-        out.write_qstr(native_qstr_objs[q])
-
     # MPY: scope_flags
     scope_flags = MP_SCOPE_FLAG_VIPERRELOC
     if len(env.full_rodata):
@@ -978,10 +969,14 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
                 kind = 0
         elif isinstance(kind, str) and kind.startswith(".bss"):
             kind = bss_const_table_idx
-        elif kind == "mp_fun_table":
+        elif kind == "mp_native_qstr_table":
             kind = 6
+        elif kind == "mp_native_obj_table":
+            kind = 7
+        elif kind == "mp_fun_table":
+            kind = 8
         else:
-            kind = 7 + kind
+            kind = 9 + kind
         assert addr % env.arch.word_size == 0, addr
         offset = addr // env.arch.word_size
         if kind == prev_kind and base == prev_base and offset == prev_offset + 1:
@@ -1023,18 +1018,14 @@ def do_preprocess(args):
         for i, q in enumerate(static_qstrs):
             print("#define %s (%u)" % (q, i + 1), file=f)
         for i, q in enumerate(sorted(qstr_vals)):
-            print("#define %s (mp_native_qstr_val_table[%d])" % (q, i), file=f)
+            print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f)
         for i, q in enumerate(sorted(qstr_objs)):
             print(
-                "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_qstr_obj_table[%d])" % (q, i),
+                "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_obj_table[%d])" % (q, i),
                 file=f,
             )
-        if args.arch == "xtensawin":
-            qstr_type = "uint32_t"  # esp32 can only read 32-bit values from IRAM
-        else:
-            qstr_type = "uint16_t"
-        print("extern const {} mp_native_qstr_val_table[];".format(qstr_type), file=f)
-        print("extern const mp_uint_t mp_native_qstr_obj_table[];", file=f)
+        print("extern const uint16_t mp_native_qstr_table[];", file=f)
+        print("extern const mp_uint_t mp_native_obj_table[];", file=f)
 
 
 def do_link(args):