bpo-31650: PEP 552 (Deterministic pycs) implementation
Python now supports checking bytecode cache up-to-dateness with a hash of the source contents rather than volatile source metadata. See the PEP for details. While a fairly straightforward idea, quite a lot of code had to be modified due to the pervasiveness of pyc implementation details in the codebase. Changes in this commit include: - The core changes to importlib to understand how to read, validate, and regenerate hash-based pycs. - Support for generating hash-based pycs in py_compile and compileall. - Modifications to our siphash implementation to support passing a custom key. We then expose it to importlib through _imp. - Updates to all places in the interpreter, standard library, and tests that manually generate or parse pyc files to grok the new format. - Support in the interpreter command line code for long options like --check-hash-based-pycs. - Tests and documentation for all of the above.

add parens for clarity
use memoryview to avoid copying
python · benjaminp · Dec 9, 2017 · benjaminp · Dec 9, 2017 · benjaminp
diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst
+      against the source at runtime to determine if the pyc needs to be
+      regenerated.
+
+   .. attribute:: UNCHECKED_HASH
diff --git a/Lib/compileall.py b/Lib/compileall.py
@@ -266,23 +284,29 @@ def main():
    if args.workers is not None:
        args.workers = args.workers or None

+    ivl_mode = args.invalidation_mode.replace('-', '_').upper()
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
-def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
-    """Validate the header of the passed-in bytecode against source_stats (if
-    given) and returning the bytecode that can be compiled by compile().
+def _classify_pyc(data, name, exc_details):
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
+    return flags
+
+
+def _validate_timestamp_pyc(data, source_stats, name, exc_details):
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
-    file."""
+
+def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
+    "Produce the data for a timestamp-based pyc."
    data = bytearray(MAGIC_NUMBER)
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
-                                source_stats=st, name=fullname,
-                                path=bytecode_path)
+                        flags = _classify_pyc(data, fullname, exc_details)
+                        bytes_data = data[16:]
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
+            'path': path,
+        }
+        _classify_pyc(data, fullname, exc_details)
+        return _compile_bytecode(data[16:], name=fullname, bytecode_path=path)
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
@@ -5,18 +5,25 @@
 from ._bootstrap import spec_from_loader
 from ._bootstrap import _find_spec
 from ._bootstrap_external import MAGIC_NUMBER
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
            except ImportError as exc:
                self.msgout(2, "raise ImportError: " + str(exc), pathname)
                raise
-            co = marshal.loads(marshal_data)
+            co = marshal.loads(data[16:])
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
+        bytecode = importlib._bootstrap_external._code_to_hash_pyc(
+            code,
+            source_hash,
+            invalidation_mode == PycInvalidationMode.CHECKED_HASH,
diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
+.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash]
+
+   Control how the generated pycs will be invalidated at runtime. The default
+   setting, ``timestamp``, means that pyc files with the source timestamp and
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
@@ -1327,6 +1330,14 @@ an :term:`importer`.
   .. versionchanged:: 3.6
      Accepts a :term:`path-like object`.

+.. function:: source_hash(source_bytes)
+
+   Return the hash of *source_bytes* as byte string. A hash-based pyc embeds the
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
+metadata.
+
+Python also supports "hash-based" cache files, which store a hash of a source
+file contents rather than its metadata. There are two variants of hash-based
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
+hash in the cache file. If a checked hash-based cache file is found to be
+invalid, Python regenerates it and writes a new checked hash-based cache
+file. For unchecked hash-based pycs, Python simply assumes the cache file is
+valid if it exists. Hash-based pyc validation behavior may be override with the
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
-    elif len(raw_timestamp) != 4:
-        message = 'reached EOF while reading timestamp in {!r}'.format(name)
+    if len(data) < 16:
+        message = 'reached EOF while reading pyc header of {!r}'.format(name)
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
+    flags = _r_long(data[4:8])
+    # Only the first two flags are defined.
+    if flags & ~0b11:
+        raise ImportError('invalid flags {!r} in {!r}'.format(flags, name),
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
+
+    """
+    if _r_long(data[8:12]) != (source_mtime & 0xFFFFFFFF):
+        message = 'bytecode is stale for {!r}'.format(name)
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
+        raise ImportError(message, **exc_details)
+    if (source_size is not None and
+        _r_long(data[12:16]) != (source_size & 0xFFFFFFFF)):
+        raise ImportError('bytecode is stale for {!r}'.format(name),
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
                                                del_source=del_source)
            test('_temp', mapping, bc_path)

    def _test_no_marshal(self, *, del_source=False):
        with util.create_modules('_temp') as mapping:
            bc_path = self.manipulate_bytecode('_temp', mapping,
-                                                lambda bc: bc[:12],
+                                                lambda bc: bc[:16],
diff --git a/Python/import.c b/Python/import.c
@@ -2216,6 +2249,14 @@ PyInit_imp(void)
    d = PyModule_GetDict(m);
    if (d == NULL)
        goto failure;
+    PyObject *pyc_mode = PyUnicode_FromString(_Py_CheckHashBasedPycsMode);
+    if (pyc_mode == NULL)
diff --git a/Modules/main.c b/Modules/main.c
@@ -98,6 +104,8 @@ static const char usage_3[] = "\
         also PYTHONWARNINGS=arg\n\
 -x     : skip first line of source, allowing use of non-Unix forms of #!cmd\n\
 -X opt : set implementation-specific option\n\
+--check-hash-based-pycs always|default|never:\n\
+    control how Python invalidates hash-based .pcy files\n\
diff --git a/Modules/perfmap.c b/Modules/perfmap.c
+};
+
+PyDoc_STRVAR(module_doc,
+"perfmap module.");
diff --git a/Modules/zipimport.c b/Modules/zipimport.c
+    if (flags != 0) {
+        // Hash-based pyc. We currently refuse to handle checked hash-based
+        // pycs. We could validate hash-based pycs against the source, but it
+        // seems likely that most people putting hash-based pycs in a zipfile