greg/cpython
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

gh-116146: Add C-API to create module from spec and initfunc (GH-139196)

Browse Source 
Co-authored-by: Kumar Aditya <kumaraditya@python.org>
Co-authored-by: Petr Viktorin <encukou@gmail.com>
Co-authored-by: Victor Stinner <vstinner@python.org>
This commit is contained in:
Itamar Oren
2025-11-14 01:43:25 -08:00
committed by GitHub
parent a4dd66275b
commit 1e4e59bb37
7 changed files with 223 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Doc/c-api/import.rst
View File

@@ -333,3 +333,24 @@ Importing Modules
strings instead of Python :class:`str` objects.
.. versionadded:: 3.14
.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void))
This function is a building block that enables embedders to implement
the :py:meth:`~importlib.abc.Loader.create_module` step of custom
static extension importers (e.g. of statically-linked extensions).
*spec* must be a :class:`~importlib.machinery.ModuleSpec` object.
*initfunc* must be an :ref:`initialization function <extension-export-hook>`,
the same as for :c:func:`PyImport_AppendInittab`.
On success, create and return a module object.
This module will not be initialized; call :c:func:`!PyModule_Exec`
to initialize it.
(Custom importers should do this in their
:py:meth:`~importlib.abc.Loader.exec_module` method.)
On error, return NULL with an exception set.
.. versionadded:: next

4
Doc/whatsnew/3.15.rst
View File

@@ -1080,6 +1080,10 @@ New features
thread state.
(Contributed by Victor Stinner in :gh:`139653`.)
* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating
a module from a *spec* and *initfunc*.
(Contributed by Itamar Oren in :gh:`116146`.)
Changed C APIs
--------------

7
Include/cpython/import.h
View File

@@ -10,6 +10,13 @@ struct _inittab {
PyAPI_DATA(struct _inittab *) PyImport_Inittab;
PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab);
// Custom importers may use this API to initialize statically linked
// extension modules directly from a spec and init function,
// without needing to go through inittab
PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc(
PyObject *spec,
PyObject *(*initfunc)(void));
struct _frozen {
const char *name; /* ASCII encoded string */
const unsigned char *code;

25
Lib/test/test_embed.py
View File

@@ -239,6 +239,31 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
lines = "\n".join(lines) + "\n"
self.assertEqual(out, lines)
def test_create_module_from_initfunc(self):
out, err = self.run_embedded_interpreter("test_create_module_from_initfunc")
if support.Py_GIL_DISABLED:
# the test imports a singlephase init extension, so it emits a warning
# under the free-threaded build
expected_runtime_warning = (
"RuntimeWarning: The global interpreter lock (GIL)"
" has been enabled to load module 'embedded_ext'"
)
filtered_err_lines = [
line
for line in err.strip().splitlines()
if expected_runtime_warning not in line
]
self.assertEqual(filtered_err_lines, [])
else:
self.assertEqual(err, "")
self.assertEqual(out,
"<module 'my_test_extension' (static-extension)>\n"
"my_test_extension.executed='yes'\n"
"my_test_extension.exec_slot_ran='yes'\n"
"<module 'embedded_ext' (static-extension)>\n"
"embedded_ext.executed='yes'\n"
)
def test_forced_io_encoding(self):
# Checks forced configuration of embedded interpreter IO streams
env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")

2
Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst Normal file
View File

@@ -0,0 +1,2 @@
Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a
module from a *spec* and *initfunc*. Patch by Itamar Oren.

111
Programs/_testembed.c
View File

@@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = {
static PyObject*
PyInit_embedded_ext(void)
{
// keep this as a single-phase initialization module;
// see test_create_module_from_initfunc
return PyModule_Create(&embedded_ext);
}
@@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void)
}
int
extension_module_exec(PyObject *mod)
{
return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes");
}
static PyModuleDef_Slot extension_slots[] = {
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
{Py_mod_exec, extension_module_exec},
{0, NULL}
};
@@ -2213,6 +2223,106 @@ static int test_repeated_init_and_inittab(void)
return 0;
}
static PyObject*
create_module(PyObject* self, PyObject* spec)
{
PyObject *name = PyObject_GetAttrString(spec, "name");
if (!name) {
return NULL;
}
if (PyUnicode_EqualToUTF8(name, "my_test_extension")) {
Py_DECREF(name);
return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension);
}
if (PyUnicode_EqualToUTF8(name, "embedded_ext")) {
Py_DECREF(name);
return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext);
}
PyErr_Format(PyExc_LookupError, "static module %R not found", name);
Py_DECREF(name);
return NULL;
}
static PyObject*
exec_module(PyObject* self, PyObject* mod)
{
if (PyModule_Exec(mod) < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyMethodDef create_static_module_methods[] = {
{"create_module", create_module, METH_O, NULL},
{"exec_module", exec_module, METH_O, NULL},
{}
};
static struct PyModuleDef create_static_module_def = {
PyModuleDef_HEAD_INIT,
.m_name = "create_static_module",
.m_size = 0,
.m_methods = create_static_module_methods,
.m_slots = extension_slots,
};
PyMODINIT_FUNC PyInit_create_static_module(void) {
return PyModuleDef_Init(&create_static_module_def);
}
static int
test_create_module_from_initfunc(void)
{
wchar_t* argv[] = {
PROGRAM_NAME,
L"-c",
// Multi-phase initialization
L"import my_test_extension;"
L"print(my_test_extension);"
L"print(f'{my_test_extension.executed=}');"
L"print(f'{my_test_extension.exec_slot_ran=}');"
// Single-phase initialization
L"import embedded_ext;"
L"print(embedded_ext);"
L"print(f'{embedded_ext.executed=}');"
};
PyConfig config;
if (PyImport_AppendInittab("create_static_module",
&PyInit_create_static_module) != 0) {
fprintf(stderr, "PyImport_AppendInittab() failed\n");
return 1;
}
PyConfig_InitPythonConfig(&config);
config.isolated = 1;
config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
init_from_config_clear(&config);
int result = PyRun_SimpleString(
"import sys\n"
"from importlib.util import spec_from_loader\n"
"import create_static_module\n"
"class StaticExtensionImporter:\n"
" _ORIGIN = \"static-extension\"\n"
" @classmethod\n"
" def find_spec(cls, fullname, path, target=None):\n"
" if fullname in {'my_test_extension', 'embedded_ext'}:\n"
" return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n"
" return None\n"
" @staticmethod\n"
" def create_module(spec):\n"
" return create_static_module.create_module(spec)\n"
" @staticmethod\n"
" def exec_module(module):\n"
" create_static_module.exec_module(module)\n"
" module.executed = 'yes'\n"
"sys.meta_path.append(StaticExtensionImporter)\n"
);
if (result < 0) {
fprintf(stderr, "PyRun_SimpleString() failed\n");
return 1;
}
return Py_RunMain();
}
static void wrap_allocator(PyMemAllocatorEx *allocator);
static void unwrap_allocator(PyMemAllocatorEx *allocator);
@@ -2396,6 +2506,7 @@ static struct TestCase TestCases[] = {
#endif
{"test_get_incomplete_frame", test_get_incomplete_frame},
{"test_gilstate_after_finalization", test_gilstate_after_finalization},
{"test_create_module_from_initfunc", test_create_module_from_initfunc},
{NULL, NULL}
};

74
Python/import.c
View File

@@ -2364,8 +2364,23 @@ is_builtin(PyObject *name)
return 0;
}
static PyModInitFunction
lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info)
{
for (struct _inittab *p = INITTAB; p->name != NULL; p++) {
if (_PyUnicode_EqualToASCIIString(info->name, p->name)) {
return (PyModInitFunction)p->initfunc;
}
}
// not found
return NULL;
}
static PyObject*
create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec)
create_builtin(
PyThreadState *tstate, PyObject *name,
PyObject *spec,
PyModInitFunction initfunc)
{
struct _Py_ext_module_loader_info info;
if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) {
@@ -2396,25 +2411,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec)
_extensions_cache_delete(info.path, info.name);
}
struct _inittab *found = NULL;
for (struct _inittab *p = INITTAB; p->name != NULL; p++) {
if (_PyUnicode_EqualToASCIIString(info.name, p->name)) {
found = p;
break;
}
}
if (found == NULL) {
// not found
mod = Py_NewRef(Py_None);
goto finally;
}
PyModInitFunction p0 = (PyModInitFunction)found->initfunc;
PyModInitFunction p0 = initfunc;
if (p0 == NULL) {
/* Cannot re-init internal module ("sys" or "builtins") */
assert(is_core_module(tstate->interp, info.name, info.path));
mod = import_add_module(tstate, info.name);
goto finally;
p0 = lookup_inittab_initfunc(&info);
if (p0 == NULL) {
/* Cannot re-init internal module ("sys" or "builtins") */
assert(is_core_module(tstate->interp, info.name, info.path));
mod = import_add_module(tstate, info.name);
goto finally;
}
}
#ifdef Py_GIL_DISABLED
@@ -2440,6 +2445,33 @@ finally:
return mod;
}
PyObject*
PyImport_CreateModuleFromInitfunc(
PyObject *spec, PyObject *(*initfunc)(void))
{
if (initfunc == NULL) {
PyErr_BadInternalCall();
return NULL;
}
PyThreadState *tstate = _PyThreadState_GET();
PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name));
if (name == NULL) {
return NULL;
}
if (!PyUnicode_Check(name)) {
PyErr_Format(PyExc_TypeError,
"spec name must be string, not %T", name);
Py_DECREF(name);
return NULL;
}
PyObject *mod = create_builtin(tstate, name, spec, initfunc);
Py_DECREF(name);
return mod;
}
/*****************************/
/* the builtin modules table */
@@ -3209,7 +3241,7 @@ bootstrap_imp(PyThreadState *tstate)
}
// Create the _imp module from its definition.
PyObject *mod = create_builtin(tstate, name, spec);
PyObject *mod = create_builtin(tstate, name, spec, NULL);
Py_CLEAR(name);
Py_DECREF(spec);
if (mod == NULL) {
@@ -4369,7 +4401,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
return NULL;
}
PyObject *mod = create_builtin(tstate, name, spec);
PyObject *mod = create_builtin(tstate, name, spec, NULL);
Py_DECREF(name);
return mod;
}