all: Switch to new preview build versioning scheme.

See https://github.com/micropython/micropython/issues/12127 for details.

Previously at the point when a release is made, we update mpconfig.h
and set a git tag. i.e. the version increments at the release.

Now the version increments immediately after the release. The workflow is:
1. Final commit in the cycle updates mpconfig.h to set (X, Y, 0, 0) (i.e.
   clear the pre-release state).
2. This commit is tagged "vX.Y.0".
3. First commit for the new cycle updates mpconfig.h to set (X, Y+1, 0, 1)
   (i.e. increment the minor version, set the pre-release state).
4. This commit is tagged "vX.Y+1.0-preview".

The idea is that a nightly build is actually a "preview" of the _next_
release. i.e. any documentation describing the current release may not
actually match the nightly build. So we use "preview" as our semver
pre-release identifier.

Changes in this commit:
 - Add MICROPY_VERSION_PRERELEASE to mpconfig.h to allow indicating that
   this is not a release version.
 - Remove unused MICROPY_VERSION integer.
 - Append "-preview" to MICROPY_VERSION_STRING when the pre-release state
   is set.
 - Update py/makeversionhdr.py to no longer generate MICROPY_GIT_HASH.
 - Remove the one place MICROPY_GIT_HASH was used (it can use
   MICROPY_GIT_TAG instead).
 - Update py/makeversionhdr.py to also understand
   MICROPY_VERSION_PRERELEASE in mpconfig.h.
 - Update py/makeversionhdr.py to convert the git-describe output into
   semver-compatible "X.Y.Z-preview.N.gHASH".
 - Update autobuild.sh to generate filenames using the new scheme.
 - Update remove_old_firmware.py to match new scheme.
 - Update mpremote's pyproject.toml to handle the "-preview" suffix in the
   tag. setuptools_scm maps to this "rc0" to match PEP440.
 - Fix docs heading where it incorrectly said "vvX.Y.Z" for release docs.

This work was funded through GitHub Sponsors.
Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

docs/conf.py

@@ -21,10 +21,12 @@ import os
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('.'))
 
-# The members of the html_context dict are available inside topindex.html
+# The MICROPY_VERSION env var should be "vX.Y.Z" (or unset).
 micropy_version = os.getenv('MICROPY_VERSION') or 'latest'
 micropy_all_versions = (os.getenv('MICROPY_ALL_VERSIONS') or 'latest').split(',')
 url_pattern = '%s/en/%%s' % (os.getenv('MICROPY_URL_PREFIX') or '/',)
+
+# The members of the html_context dict are available inside topindex.html
 html_context = {
     'cur_version':micropy_version,
     'all_versions':[

docs/templates/layout.html

@@ -9,7 +9,7 @@
   {% if is_release %}
     <div class="wy-alert wy-alert-danger">
       <p>
-        This is the v{{ release }} version of the MicroPython
+        This is the {{ release }} version of the MicroPython
         documentation. The <a href="/en/latest/{{ pagename }}.html">latest
         development version</a> of this page may be more current.
       </p>

ports/cc3200/tools/update-wipy.py

@@ -152,9 +152,9 @@ def verify_update(args):
             with open(tag_file_path) as tag_file:
                 for line in tag_file:
                     bline = bytes(line, "ascii")
-                    if b"MICROPY_GIT_HASH" in bline:
+                    if b"MICROPY_GIT_TAG" in bline:
                         bline = (
-                            bline.lstrip(b"#define MICROPY_GIT_HASH ")
+                            bline.lstrip(b"#define MICROPY_GIT_TAG ")
                             .replace(b'"', b"")
                             .replace(b"\r", b"")
                             .replace(b"\n", b"")

py/makeversionhdr.py

@@ -13,6 +13,14 @@ import datetime
 import subprocess
 
 
+# The MicroPython repository tags a release commit as "vX.Y.Z", and the commit
+# immediately following as "vX.(Y+1).Z-preview".
+# This function will return:
+#   "vX.Y.Z" -- building at the release commit
+#   "vX.Y.Z-preview" -- building at the first commit in the next cycle
+#   "vX.Y.Z-preview.N.gHASH" -- building at any subsequent commit in the cycle
+#   "vX.Y.Z-preview.N.gHASH.dirty" -- building at any subsequent commit in the cycle
+#                                     with local changes
 def get_version_info_from_git(repo_path):
     # Python 2.6 doesn't have check_output, so check for that
     try:
@@ -29,47 +37,33 @@ def get_version_info_from_git(repo_path):
             stderr=subprocess.STDOUT,
             universal_newlines=True,
         ).strip()
-    except subprocess.CalledProcessError as er:
-        if er.returncode == 128:
-            # git exit code of 128 means no repository found
-            return None
-        git_tag = ""
-    except OSError:
-        return None
-    try:
-        git_hash = subprocess.check_output(
-            ["git", "rev-parse", "--short", "HEAD"],
-            cwd=repo_path,
-            stderr=subprocess.STDOUT,
-            universal_newlines=True,
-        ).strip()
+        # Turn git-describe's output into semver compatible (dot-separated
+        # identifiers inside the prerelease field).
+        git_tag = git_tag.split("-", 1)
+        if len(git_tag) == 1:
+            return git_tag[0]
+        else:
+            return git_tag[0] + "-" + git_tag[1].replace("-", ".")
     except subprocess.CalledProcessError:
-        git_hash = "unknown"
-    except OSError:
         return None
-
-    try:
-        # Check if there are any modified files.
-        subprocess.check_call(
-            ["git", "diff", "--no-ext-diff", "--quiet", "--exit-code"],
-            cwd=repo_path,
-            stderr=subprocess.STDOUT,
-        )
-        # Check if there are any staged files.
-        subprocess.check_call(
-            ["git", "diff-index", "--cached", "--quiet", "HEAD", "--"],
-            cwd=repo_path,
-            stderr=subprocess.STDOUT,
-        )
-    except subprocess.CalledProcessError:
-        git_hash += "-dirty"
     except OSError:
         return None
 
-    return git_tag, git_hash
-
 
+# When building from a source tarball (or any situation where the git repo
+# isn't available), this function will use the info in mpconfig.h as a
+# fallback. The release commit sets MICROPY_VERSION_PRERELEASE to 0, and the
+# commit immediately following increments MICROPY_VERSION_MINOR and sets
+# MICROPY_VERSION_PRERELEASE back to 1.
+# This function will return:
+#    "vX.Y.Z" -- building at the release commit
+#    "vX.Y.Z-preview" -- building at any other commit
 def get_version_info_from_mpconfig(repo_path):
+    print(
+        "makeversionhdr.py: Warning: No git repo or tag info available, falling back to mpconfig.h version info.",
+        file=sys.stderr,
+    )
+
     with open(os.path.join(repo_path, "py", "mpconfig.h")) as f:
         for line in f:
             if line.startswith("#define MICROPY_VERSION_MAJOR "):
@@ -78,21 +72,30 @@ def get_version_info_from_mpconfig(repo_path):
                 ver_minor = int(line.strip().split()[2])
             elif line.startswith("#define MICROPY_VERSION_MICRO "):
                 ver_micro = int(line.strip().split()[2])
-                git_tag = "v%d.%d.%d" % (ver_major, ver_minor, ver_micro)
-                return git_tag, "<no hash>"
+            elif line.startswith("#define MICROPY_VERSION_PRERELEASE "):
+                ver_prerelease = int(line.strip().split()[2])
+                git_tag = "v%d.%d.%d%s" % (
+                    ver_major,
+                    ver_minor,
+                    ver_micro,
+                    "-preview" if ver_prerelease else "",
+                )
+                return git_tag
     return None
 
 
 def make_version_header(repo_path, filename):
-    info = None
+    git_tag = None
     if "MICROPY_GIT_TAG" in os.environ:
-        info = [os.environ["MICROPY_GIT_TAG"], os.environ["MICROPY_GIT_HASH"]]
-    if info is None:
-        info = get_version_info_from_git(repo_path)
-    if info is None:
-        info = get_version_info_from_mpconfig(repo_path)
+        git_tag = os.environ["MICROPY_GIT_TAG"]
+    if git_tag is None:
+        git_tag = get_version_info_from_git(repo_path)
+    if git_tag is None:
+        git_tag = get_version_info_from_mpconfig(repo_path)
 
-    git_tag, git_hash = info
+    if not git_tag:
+        print("makeversionhdr.py: Error: No version information available.")
+        sys.exit(1)
 
     build_date = datetime.date.today()
     if "SOURCE_DATE_EPOCH" in os.environ:
@@ -104,11 +107,9 @@ def make_version_header(repo_path, filename):
     file_data = """\
 // This file was generated by py/makeversionhdr.py
 #define MICROPY_GIT_TAG "%s"
-#define MICROPY_GIT_HASH "%s"
 #define MICROPY_BUILD_DATE "%s"
 """ % (
         git_tag,
-        git_hash,
         build_date.strftime("%Y-%m-%d"),
     )
 

py/modsys.c

@@ -59,16 +59,24 @@ const mp_print_t mp_sys_stdout_print = {&mp_sys_stdout_obj, mp_stream_write_adap
 STATIC const MP_DEFINE_STR_OBJ(mp_sys_version_obj, "3.4.0; " MICROPY_BANNER_NAME_AND_VERSION);
 
 // version_info - Python language version that this implementation conforms to, as a tuple of ints
-#define I(n) MP_OBJ_NEW_SMALL_INT(n)
-// TODO: CPython is now at 5-element array, but save 2 els so far...
-STATIC const mp_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {I(3), I(4), I(0)}};
+// TODO: CPython is now at 5-element array (major, minor, micro, releaselevel, serial), but save 2 els so far...
+STATIC const mp_rom_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {MP_ROM_INT(3), MP_ROM_INT(4), MP_ROM_INT(0)}};
 
 // sys.implementation object
 // this holds the MicroPython version
-STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = {
+STATIC const mp_rom_obj_tuple_t mp_sys_implementation_version_info_obj = {
     {&mp_type_tuple},
-    3,
-    { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) }
+    4,
+    {
+        MP_ROM_INT(MICROPY_VERSION_MAJOR),
+        MP_ROM_INT(MICROPY_VERSION_MINOR),
+        MP_ROM_INT(MICROPY_VERSION_MICRO),
+        #if MICROPY_VERSION_PRERELEASE
+        MP_ROM_QSTR(MP_QSTR_preview),
+        #else
+        MP_ROM_QSTR(MP_QSTR_),
+        #endif
+    }
 };
 STATIC const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER_MACHINE);
 #if MICROPY_PERSISTENT_CODE_LOAD

py/mpconfig.h

@@ -26,22 +26,32 @@
 #ifndef MICROPY_INCLUDED_PY_MPCONFIG_H
 #define MICROPY_INCLUDED_PY_MPCONFIG_H
 
-// Current version of MicroPython
+// Current version of MicroPython. This is used by sys.implementation.version
+// as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags
+// are unavailable.
 #define MICROPY_VERSION_MAJOR 1
-#define MICROPY_VERSION_MINOR 21
+#define MICROPY_VERSION_MINOR 22
 #define MICROPY_VERSION_MICRO 0
-
-// Combined version as a 32-bit number for convenience
-#define MICROPY_VERSION ( \
-    MICROPY_VERSION_MAJOR << 16 \
-        | MICROPY_VERSION_MINOR << 8 \
-        | MICROPY_VERSION_MICRO)
-
-// String version
-#define MICROPY_VERSION_STRING \
+#define MICROPY_VERSION_PRERELEASE 1
+
+// Combined version as a 32-bit number for convenience to allow version
+// comparison. Doesn't include prerelease state.
+// e.g. #if MICROPY_VERSION < MICROPY_MAKE_VERSION(1, 22, 0)
+#define MICROPY_MAKE_VERSION(major, minor, patch) (major << 16 | minor << 8 | patch)
+#define MICROPY_VERSION MICROPY_MAKE_VERSION(MICROPY_VERSION_MAJOR, MICROPY_VERSION_MINOR, MICROPY_VERSION_MICRO)
+
+// String version. This is only used directly for platform.platform and
+// os.uname().release. All other version info available in the firmware (e.g.
+// the REPL banner) comes from MICROPY_GIT_TAG.
+#define MICROPY_VERSION_STRING_BASE \
     MP_STRINGIFY(MICROPY_VERSION_MAJOR) "." \
     MP_STRINGIFY(MICROPY_VERSION_MINOR) "." \
     MP_STRINGIFY(MICROPY_VERSION_MICRO)
+#if MICROPY_VERSION_PRERELEASE
+#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE "-preview"
+#else
+#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE
+#endif
 
 // This file contains default configuration settings for MicroPython.
 // You can override any of the options below using mpconfigport.h file

tools/autobuild/autobuild.sh

@@ -54,9 +54,19 @@ pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}
 make -C mpy-cross
 
 # make the firmware tag
+# final filename will be <BOARD><-VARIANT>-<DATE>-v<SEMVER>.ext
+# where SEMVER is vX.Y.Z or vX.Y.Z-preview.N.gHASH or vX.Y.Z-preview.N.gHASH.dirty
 FW_DATE=$(date '+%Y%m%d')
-FW_GIT="$(git describe --dirty || echo unknown)"
-FW_TAG="-$FW_DATE-unstable-$FW_GIT"
+# same logic as makeversionhdr.py, convert git-describe output into semver-compatible
+FW_GIT_TAG="$(git describe --tags --dirty --always --match 'v[1-9].*')"
+FW_SEMVER_MAJOR_MINOR_PATCH="$(echo $FW_GIT_TAG | cut -d'-' -f1)"
+FW_SEMVER_PRERELEASE="$(echo $FW_GIT_TAG | cut -s -d'-' -f2-)"
+if [ -z "$FW_SEMVER_PRERELEASE" ]; then
+    FW_SEMVER="$FW_SEMVER_MAJOR_MINOR_PATCH"
+else
+    FW_SEMVER="$FW_SEMVER_MAJOR_MINOR_PATCH-$(echo $FW_SEMVER_PRERELEASE | tr - .)"
+fi
+FW_TAG="-$FW_DATE-$FW_SEMVER"
 
 # build new firmware
 cd ports/cc3200

tools/autobuild/remove_old_firmware.py

@@ -14,7 +14,7 @@ def main():
 
     # SSH to get list of existing files.
     p = subprocess.run(
-        ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-unstable-v\\*"],
+        ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-preview.\\*"],
         capture_output=True,
     )
     if p.returncode != 0:
@@ -26,31 +26,33 @@ def main():
     boards = {}
     for file in all_files:
         m = re.match(
-            rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-unstable-(v[0-9.-]+-g[0-9a-f]+).",
+            rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-(v[0-9.]+)-preview.([0-9]+).g[0-9a-f]+.",
             file,
         )
         if not m:
             continue
-        dir, board, date, version = m.groups()
+        dir, board, date, version, ncommits = m.groups()
         if board not in boards:
             boards[board] = {}
-        if (date, version) not in boards[board]:
-            boards[board][(date, version)] = []
-        boards[board][(date, version)].append(file)
+        if (date, version, ncommits) not in boards[board]:
+            boards[board][(date, version, ncommits)] = []
+        boards[board][(date, version, ncommits)].append(file)
 
     # Collect files to remove based on date and version.
     remove = []
     for board in boards.values():
-        filelist = [(date, version, files) for (date, version), files in board.items()]
+        filelist = [
+            (date, version, ncommits, files) for (date, version, ncommits), files in board.items()
+        ]
         filelist.sort(reverse=True)
         keep = []
-        for date, version, files in filelist:
-            if keep and version == keep[-1]:
+        for date, version, ncommits, files in filelist:
+            if keep and (version, ncommits) == keep[-1]:
                 remove.extend(files)
             elif len(keep) >= NUM_KEEP_PER_BOARD:
                 remove.extend(files)
             else:
-                keep.append(version)
+                keep.append((version, ncommits))
 
     if DEBUG:
         all_files.sort(reverse=True)

tools/mpremote/pyproject.toml

@@ -36,9 +36,14 @@ mpremote = "mpremote.main:main"
 [tool.hatch.metadata.hooks.requirements_txt]
 files = ["requirements.txt"]
 
+# This will be PEP-440 normalised into either:
+# mpremote-X.Y.Z (on vX.Y.Z release tag)
+# mpremote-X.Y.Zrc0 (on vX.Y.Z-preview tag, i.e. first commit in the cycle)
+# mpremote-X.Y.Zrc0.postN+gHASH (N commits past vX.Y.Z-preview tag)
+# mpremote-X.Y.Zrc0.postN+gHASH.dDATE (N commits past vX.Y.Z-preview tag, dirty)
 [tool.hatch.version]
 source = "vcs"
-tag-pattern = "(?P<version>v(\\d+).(\\d+).(\\d+))"
+tag-pattern = "(?P<version>v(\\d+).(\\d+).(\\d+)(-preview)?)"
 raw-options = { root = "../..", version_scheme = "post-release" }
 
 [tool.hatch.build]