Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions Doc/library/sys.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1483,6 +1483,27 @@
They hold the legacy representation of ``sys.last_exc``, as returned
from :func:`exc_info` above.


.. data:: lazy_modules

A read-only :class:`frozendict` snapshot of the lazy import registry for the
current interpreter. Each key is a module name string; the corresponding
value is a :class:`frozenset` of the child attribute names that were
registered as lazy imports from that module. A top-level lazy import (e.g.
``lazy import json``) is represented by an empty :class:`frozenset`.

A fresh snapshot is constructed on every attribute access, so the mapping
reflects the state of lazy imports at the time of access. The snapshot is
immutable; modifications must go through :func:`set_lazy_imports`.

The live, mutable registry is available as the private :data:`sys._lazy_modules`

Check warning on line 1499 in Doc/library/sys.rst

View workflow job for this annotation

GitHub Actions / Docs / Docs

py:data reference target not found: sys._lazy_modules [ref.data]
for internal use.

See also :func:`set_lazy_imports`, :func:`get_lazy_imports`, and :pep:`810`.

.. versionadded:: 3.15


.. data:: maxsize

An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
extern PyObject * _PyImport_InitLazyModules(
PyInterpreterState *interp);
extern void _PyImport_ClearLazyModules(PyInterpreterState *interp);
extern PyObject * _PyImport_GetLazyModulesSnapshot(
PyInterpreterState *interp);

extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
extern int _PyImport_IsDefaultImportFunc(
Expand Down
4 changes: 3 additions & 1 deletion Lib/test/test_builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -2132,7 +2132,9 @@ def getDict(self):

def test_vars(self):
self.assertEqual(set(vars()), set(dir()))
self.assertEqual(set(vars(sys)), set(dir(sys)))
# sys.lazy_modules is a virtual attribute served by sys.__getattr__,
# so it appears in dir() but not in vars().
self.assertEqual(set(vars(sys)) | {'lazy_modules'}, set(dir(sys)))
self.assertEqual(self.get_vars_f0(), {})
self.assertEqual(self.get_vars_f2(), {'a': 1, 'b': 2})
self.assertRaises(TypeError, vars, 42, 42)
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_inspect/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -6179,6 +6179,9 @@ def test_sys_module_has_signatures(self):
no_signature = {'getsizeof', 'set_asyncgen_hooks'}
no_signature |= {name for name in ['getobjects']
if hasattr(sys, name)}
# sys.__dir__ and sys.__getattr__ are plain METH_NOARGS/METH_VARARGS
# builtins without Argument Clinic text signatures.
no_signature |= {'__dir__', '__getattr__'}
self._test_module_has_signatures(sys, no_signature)

def test_abc_module_has_signatures(self):
Expand Down
37 changes: 30 additions & 7 deletions Lib/test/test_lazy_import/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def tearDown(self):

sys.set_lazy_imports_filter(None)
sys.set_lazy_imports("normal")
sys.lazy_modules.clear()
sys._lazy_modules.clear()

def test_basic_unused(self):
"""Lazy imported module should not be loaded if never accessed."""
Expand Down Expand Up @@ -484,9 +484,30 @@ def my_filter(name):
sys.set_lazy_imports_filter(my_filter)
self.assertIs(sys.get_lazy_imports_filter(), my_filter)

def test_lazy_modules_attribute_is_dict(self):
"""sys.lazy_modules should be a dict per PEP 810."""
self.assertIsInstance(sys.lazy_modules, dict)
def test_lazy_modules_attribute_is_frozendict(self):
"""sys.lazy_modules should be an immutable frozendict mapping."""
snapshot = sys.lazy_modules
self.assertIsInstance(snapshot, frozendict)
for value in snapshot.values():
self.assertIsInstance(value, frozenset)

def test_lazy_modules_returns_fresh_mapping(self):
"""Each access of sys.lazy_modules should return a fresh mapping."""
first = sys.lazy_modules
second = sys.lazy_modules
# Snapshots are independent objects, even though they may compare equal.
self.assertIsNot(first, second)
self.assertEqual(first, second)

def test_underscore_lazy_modules_is_live_dict(self):
"""sys._lazy_modules should be the live, mutable registry."""
registry = sys._lazy_modules
self.assertIsInstance(registry, dict)
self.assertNotIsInstance(registry, frozendict)
# Same identity across accesses (it is the registry itself).
self.assertIs(sys._lazy_modules, registry)
# Snapshot reflects the live registry contents.
self.assertEqual(dict(sys.lazy_modules), dict(registry))

@support.requires_subprocess()
def test_lazy_modules_tracks_lazy_imports(self):
Expand Down Expand Up @@ -966,8 +987,8 @@ def test_module_added_to_lazy_modules_on_lazy_import(self):

def test_lazy_modules_is_per_interpreter(self):
"""Each interpreter should have independent sys.lazy_modules."""
# Basic test that sys.lazy_modules exists and is a dict
self.assertIsInstance(sys.lazy_modules, dict)
# Basic test that sys.lazy_modules exists and is an immutable mapping.
self.assertIsInstance(sys.lazy_modules, frozendict)

def test_lazy_module_without_children_is_tracked(self):
code = textwrap.dedent("""
Expand Down Expand Up @@ -1804,7 +1825,9 @@ def create_lazy_imports(idx):
t.join()

assert not errors, f"Errors: {errors}"
assert isinstance(sys.lazy_modules, dict), "sys.lazy_modules is not a dict"
assert isinstance(sys.lazy_modules, frozendict), (
"sys.lazy_modules is not a frozendict"
)
print("OK")
""")

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
:data:`sys.lazy_modules` is now a read-only :class:`frozendict` mapping
each module's fully qualified name to the :class:`frozenset` of its
lazily-imported submodule names, instead of a live mutable :class:`dict`
of :class:`set` objects.
42 changes: 42 additions & 0 deletions Python/import.c
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,48 @@ _PyImport_ClearLazyModules(PyInterpreterState *interp)
Py_CLEAR(LAZY_MODULES(interp));
}

PyObject *
_PyImport_GetLazyModulesSnapshot(PyInterpreterState *interp)
{
PyObject *lazy_modules = LAZY_MODULES(interp);
if (lazy_modules == NULL) {
return PyFrozenDict_New(NULL);
}

PyObject *tmp = PyDict_New();
if (tmp == NULL) {
return NULL;
}

int err = 0;
Py_BEGIN_CRITICAL_SECTION(lazy_modules);
Py_ssize_t pos = 0;
PyObject *key, *value;
while (PyDict_Next(lazy_modules, &pos, &key, &value)) {
PyObject *frozen = PyFrozenSet_New(value);
if (frozen == NULL) {
err = -1;
break;
}
if (PyDict_SetItem(tmp, key, frozen) < 0) {
Py_DECREF(frozen);
err = -1;
break;
}
Py_DECREF(frozen);
}
Py_END_CRITICAL_SECTION();

if (err < 0) {
Py_DECREF(tmp);
return NULL;
}

PyObject *snapshot = PyFrozenDict_New(tmp);
Py_DECREF(tmp);
return snapshot;
}

static int
import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name)
{
Expand Down
43 changes: 41 additions & 2 deletions Python/sysmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -2909,8 +2909,45 @@ sys_get_lazy_imports_impl(PyObject *module)
}
}

static PyObject *
sys_getattr(PyObject *self, PyObject *args)
{
PyObject *name;
if (!PyArg_UnpackTuple(args, "__getattr__", 1, 1, &name)) {
return NULL;
}

if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "lazy_modules")) {
PyInterpreterState *interp = _PyInterpreterState_GET();
return _PyImport_GetLazyModulesSnapshot(interp);
}

PyErr_Format(PyExc_AttributeError,
"module 'sys' has no attribute %R", name);
return NULL;
}

static PyObject *
sys_dir(PyObject *self, PyObject *Py_UNUSED(ignored))
{
PyObject *names = PyMapping_Keys(((PyModuleObject *)self)->md_dict);
if (names == NULL) {
return NULL;
}
PyObject *lazy = PyUnicode_FromString("lazy_modules");
int err = lazy ? PyList_Append(names, lazy) : -1;
Py_XDECREF(lazy);
if (err < 0) {
Py_DECREF(names);
return NULL;
}
return names;
}

static PyMethodDef sys_methods[] = {
/* Might as well keep this in alphabetic order */
{"__dir__", sys_dir, METH_NOARGS, "Module __dir__"},
{"__getattr__", sys_getattr, METH_VARARGS, "Module __getattr__"},
SYS_ADDAUDITHOOK_METHODDEF
SYS_AUDIT_METHODDEF
{"breakpointhook", _PyCFunction_CAST(sys_breakpointhook),
Expand Down Expand Up @@ -4353,12 +4390,14 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
goto error;
}

// The live lazy import registry is exposed (undocumented) as
// ``sys._lazy_modules``. The public ``sys.lazy_modules`` is built on
// each access by ``sys.__getattr__`` (see ``sys_getattr``).
PyObject *lazy_modules = _PyImport_InitLazyModules(interp); // borrowed reference
if (lazy_modules == NULL) {
goto error;
}

if (PyDict_SetItemString(sysdict, "lazy_modules", lazy_modules) < 0) {
if (PyDict_SetItemString(sysdict, "_lazy_modules", lazy_modules) < 0) {
goto error;
}

Expand Down
Loading