How is super() in Python 3 implemented?
How is super() implemented? Here's the code for python3.3:
/* Cooperative 'super' */
typedef struct {
PyObject_HEAD
PyTypeObject *type;
PyObject *obj;
PyTypeObject *obj_type;
} superobject;
static PyMemberDef super_members[] = {
{"__thisclass__", T_OBJECT, offsetof(superobject, type), READONLY,
"the class invoking super()"},
{"__self__", T_OBJECT, offsetof(superobject, obj), READONLY,
"the instance invoking super(); may be None"},
{"__self_class__", T_OBJECT, offsetof(superobject, obj_type), READONLY,
"the type of the instance invoking super(); may be None"},
{0}
};
static void
super_dealloc(PyObject *self)
{
superobject *su = (superobject *)self;
_PyObject_GC_UNTRACK(self);
Py_XDECREF(su->obj);
Py_XDECREF(su->type);
Py_XDECREF(su->obj_type);
Py_TYPE(self)->tp_free(self);
}
static PyObject *
super_repr(PyObject *self)
{
superobject *su = (superobject *)self;
if (su->obj_type)
return PyUnicode_FromFormat(
"<super: <class '%s'>, <%s object>>",
su->type ? su->type->tp_name : "NULL",
su->obj_type->tp_name);
else
return PyUnicode_FromFormat(
"<super: <class '%s'>, NULL>",
su->type ? su->type->tp_name : "NULL");
}
static PyObject *
super_getattro(PyObject *self, PyObject *name)
{
superobject *su = (superobject *)self;
int skip = su->obj_type == NULL;
if (!skip) {
/* We want __class__ to return the class of the super object
(i.e. super, or a subclass), not the class of su->obj. */
skip = (PyUnicode_Check(name) &&
PyUnicode_GET_LENGTH(name) == 9 &&
PyUnicode_CompareWithASCIIString(name, "__class__") == 0);
}
if (!skip) {
PyObject *mro, *res, *tmp, *dict;
PyTypeObject *starttype;
descrgetfunc f;
Py_ssize_t i, n;
starttype = su->obj_type;
mro = starttype->tp_mro;
if (mro == NULL)
n = 0;
else {
assert(PyTuple_Check(mro));
n = PyTuple_GET_SIZE(mro);
}
for (i = 0; i < n; i++) {
if ((PyObject *)(su->type) == PyTuple_GET_ITEM(mro, i))
break;
}
i++;
res = NULL;
/* keep a strong reference to mro because starttype->tp_mro can be
replaced during PyDict_GetItem(dict, name) */
Py_INCREF(mro);
for (; i < n; i++) {
tmp = PyTuple_GET_ITEM(mro, i);
if (PyType_Check(tmp))
dict = ((PyTypeObject *)tmp)->tp_dict;
else
continue;
res = PyDict_GetItem(dict, name);
if (res != NULL) {
Py_INCREF(res);
f = Py_TYPE(res)->tp_descr_get;
if (f != NULL) {
tmp = f(res,
/* Only pass 'obj' param if
this is instance-mode super
(See SF ID #743627)
*/
(su->obj == (PyObject *)
su->obj_type
? (PyObject *)NULL
: su->obj),
(PyObject *)starttype);
Py_DECREF(res);
res = tmp;
}
Py_DECREF(mro);
return res;
}
}
Py_DECREF(mro);
}
return PyObject_GenericGetAttr(self, name);
}
static PyTypeObject *
supercheck(PyTypeObject *type, PyObject *obj)
{
/* Check that a super() call makes sense. Return a type object.
obj can be a class, or an instance of one:
- If it is a class, it must be a subclass of 'type'. This case is
used for class methods; the return value is obj.
- If it is an instance, it must be an instance of 'type'. This is
the normal case; the return value is obj.__class__.
But... when obj is an instance, we want to allow for the case where
Py_TYPE(obj) is not a subclass of type, but obj.__class__ is!
This will allow using super() with a proxy for obj.
*/
/* Check for first bullet above (special case) */
if (PyType_Check(obj) && PyType_IsSubtype((PyTypeObject *)obj, type)) {
Py_INCREF(obj);
return (PyTypeObject *)obj;
}
/* Normal case */
if (PyType_IsSubtype(Py_TYPE(obj), type)) {
Py_INCREF(Py_TYPE(obj));
return Py_TYPE(obj);
}
else {
/* Try the slow way */
PyObject *class_attr;
class_attr = _PyObject_GetAttrId(obj, &PyId___class__);
if (class_attr != NULL &&
PyType_Check(class_attr) &&
(PyTypeObject *)class_attr != Py_TYPE(obj))
{
int ok = PyType_IsSubtype(
(PyTypeObject *)class_attr, type);
if (ok)
return (PyTypeObject *)class_attr;
}
if (class_attr == NULL)
PyErr_Clear();
else
Py_DECREF(class_attr);
}
PyErr_SetString(PyExc_TypeError,
"super(type, obj): "
"obj must be an instance or subtype of type");
return NULL;
}
static PyObject *
super_descr_get(PyObject *self, PyObject *obj, PyObject *type)
{
superobject *su = (superobject *)self;
superobject *newobj;
if (obj == NULL || obj == Py_None || su->obj != NULL) {
/* Not binding to an object, or already bound */
Py_INCREF(self);
return self;
}
if (Py_TYPE(su) != &PySuper_Type)
/* If su is an instance of a (strict) subclass of super,
call its type */
return PyObject_CallFunctionObjArgs((PyObject *)Py_TYPE(su),
su->type, obj, NULL);
else {
/* Inline the common case */
PyTypeObject *obj_type = supercheck(su->type, obj);
if (obj_type == NULL)
return NULL;
newobj = (superobject *)PySuper_Type.tp_new(&PySuper_Type,
NULL, NULL);
if (newobj == NULL)
return NULL;
Py_INCREF(su->type);
Py_INCREF(obj);
newobj->type = su->type;
newobj->obj = obj;
newobj->obj_type = obj_type;
return (PyObject *)newobj;
}
}
static int
super_init(PyObject *self, PyObject *args, PyObject *kwds)
{
superobject *su = (superobject *)self;
PyTypeObject *type = NULL;
PyObject *obj = NULL;
PyTypeObject *obj_type = NULL;
if (!_PyArg_NoKeywords("super", kwds))
return -1;
if (!PyArg_ParseTuple(args, "|O!O:super", &PyType_Type, &type, &obj))
return -1;
if (type == NULL) {
/* Call super(), without args -- fill in from __class__
and first local variable on the stack. */
PyFrameObject *f = PyThreadState_GET()->frame;
PyCodeObject *co = f->f_code;
Py_ssize_t i, n;
if (co == NULL) {
PyErr_SetString(PyExc_SystemError,
"super(): no code object");
return -1;
}
if (co->co_argcount == 0) {
PyErr_SetString(PyExc_SystemError,
"super(): no arguments");
return -1;
}
obj = f->f_localsplus[0];
if (obj == NULL) {
PyErr_SetString(PyExc_SystemError,
"super(): arg[0] deleted");
return -1;
}
if (co->co_freevars == NULL)
n = 0;
else {
assert(PyTuple_Check(co->co_freevars));
n = PyTuple_GET_SIZE(co->co_freevars);
}
for (i = 0; i < n; i++) {
PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
assert(PyUnicode_Check(name));
if (!PyUnicode_CompareWithASCIIString(name,
"__class__")) {
Py_ssize_t index = co->co_nlocals +
PyTuple_GET_SIZE(co->co_cellvars) + i;
PyObject *cell = f->f_localsplus[index];
if (cell == NULL || !PyCell_Check(cell)) {
PyErr_SetString(PyExc_SystemError,
"super(): bad __class__ cell");
return -1;
}
type = (PyTypeObject *) PyCell_GET(cell);
if (type == NULL) {
PyErr_SetString(PyExc_SystemError,
"super(): empty __class__ cell");
return -1;
}
if (!PyType_Check(type)) {
PyErr_Format(PyExc_SystemError,
"super(): __class__ is not a type (%s)",
Py_TYPE(type)->tp_name);
return -1;
}
break;
}
}
if (type == NULL) {
PyErr_SetString(PyExc_SystemError,
"super(): __class__ cell not found");
return -1;
}
}
if (obj == Py_None)
obj = NULL;
if (obj != NULL) {
obj_type = supercheck(type, obj);
if (obj_type == NULL)
return -1;
Py_INCREF(obj);
}
Py_INCREF(type);
su->type = type;
su->obj = obj;
su->obj_type = obj_type;
return 0;
}
PyDoc_STRVAR(super_doc,
"super() -> same as super(__class__, <first argument>)\n"
"super(type) -> unbound super object\n"
"super(type, obj) -> bound super object; requires isinstance(obj, type)\n"
"super(type, type2) -> bound super object; requires issubclass(type2, type)\n"
"Typical use to call a cooperative superclass method:\n"
"class C(B):\n"
" def meth(self, arg):\n"
" super().meth(arg)\n"
"This works for class methods too:\n"
"class C(B):\n"
" @classmethod\n"
" def cmeth(cls, arg):\n"
" super().cmeth(arg)\n");
static int
super_traverse(PyObject *self, visitproc visit, void *arg)
{
superobject *su = (superobject *)self;
Py_VISIT(su->obj);
Py_VISIT(su->type);
Py_VISIT(su->obj_type);
return 0;
}
PyTypeObject PySuper_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"super", /* tp_name */
sizeof(superobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
super_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
super_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
super_getattro, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
super_doc, /* tp_doc */
super_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
super_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
super_descr_get, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
super_init, /* tp_init */
PyType_GenericAlloc, /* tp_alloc */
PyType_GenericNew, /* tp_new */
PyObject_GC_Del, /* tp_free */
};
You can see in the super_init at some point there is the check type == NULL and then it raises the error that you see. It is not normal to have NULL s around, so there's probably a bug somewhere in super(and note that super already had bugs in previous releases). At least I'd thought that the cases in which SystemError is raised should be triggered only due to some "internal" failure of the interpreter or some other C code and not from python code.
Also, this did not happen only to you, you can find a post in which this behaviour is considered a bug.
TL;DR: This "empty __class__ cell" error will happen when the metaclass tries to call a method in the defined class (or instantiate it) before it is done with its __new__ and __init__,and the called method uses super. The error will also happen if one writes a call to super() in a function defined outside of a class body, and tries to add this method to an existing class and use it. (update: this behavior has been fixed in Python 3.6)
Python 3 super makes an implicit reference to a "magic" __class__[*] name which behaves as a cell variable in the namespace of each class method.
This variable is created automatically at the end of the class creation mechanism - i.e. whenever there is a class body in Python, the metaclass's __new__ and __init__ are run - when __init__ finishes, the __class__ cell is populated and made available to the class's methods.
What is going on here, is that likely (I have not looked at all the code) in the PluginMeta initialization code, the class's __init__ is called, before the end of the metaclass __init__ - since one of the points of this metaclass is handling singletons - what is likely to happen is that the metaclass mechanism is instantiating the single-instance and populating __instance__ before returning from the metaclass' __init__. The implicit __class__ used by super does not exist at this point.
Thus, referencing the superclass by the hardcoded name, as one had to do prior to super in Python2 will work - and is the best way to achieve what you want there.
*- This is not the self.__class__ attribute of the instance, it is a __class__ variable actually available inside methods:
class A:
def a(self):
print ("Instance's class: {}, "
"actual class where this line is coded: {}".format(
self.__class__, __class__))
class B(A):
pass
And running this we have:
>>> B().a()
Instance's class: <class '__main__.B'>, actual class where this line is coded: <class '__main__.A'>
>>>
From the Python datamodel:
__class__is an implicit closure reference created by the compiler if any methods in a class body refer to either__class__orsuper. This allows the zero argument form ofsuper()to correctly identify the class being defined based on lexical scoping, while the class or instance that was used to make the current call is identified based on the first argument passed to the method.
For more details, please check PEP 3135