diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 63bc0803e79..341915767f9 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1882,7 +1882,6 @@ class D(dict): D.__getitem__ = dict.__getitem__ self.assertIs(d[None], None) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != def test_tuple_subclass_as_bases(self): # gh-132176: it used to crash on using # tuple subclass for as base classes. diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index d48639b2c11..3eb229fdb99 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -308,6 +308,23 @@ impl PyTuple { } impl PyTuple> { + pub(crate) fn new_ref_typed_with_type( + elements: Vec>, + tuple_type: PyTypeRef, + ) -> PyRef { + // SAFETY: PyRef has the same layout as PyObjectRef. + unsafe { + let elements: Vec = + core::mem::transmute::>, Vec>(elements); + let tuple = PyRef::new_ref( + PyTuple::new_unchecked(elements.into_boxed_slice()), + tuple_type, + None, + ); + core::mem::transmute::, PyRef>(tuple) + } + } + pub fn new_ref_typed(elements: Vec>, ctx: &Context) -> PyRef { // SAFETY: PyRef has the same layout as PyObjectRef unsafe { diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index c27c6475f42..42766e30f4f 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -42,11 +42,13 @@ use num_traits::ToPrimitive; use rustpython_common::wtf8::Wtf8; use std::collections::HashSet; +pub(crate) type PyTypeTupleRef = PyRef>; + #[pyclass(module = false, name = "type", traverse = "manual")] pub struct PyType { /// tp_base. Written under the type lock (see `set_bases`); read lock-free. pub base: PyAtomicRef>, - pub bases: PyRwLock>, + pub bases: PyRwLock, pub mro: PyRwLock>, pub subclasses: PyRwLock>>, pub attributes: PyRwLock, @@ -242,7 +244,7 @@ unsafe impl crate::object::Traverse for PyType { if let Some(base) = self.base.deref() { tracer_fn(base.as_object()); } - self.bases.traverse(tracer_fn); + tracer_fn(self.bases.read_recursive().as_untyped().as_object()); self.mro.traverse(tracer_fn); self.subclasses.traverse(tracer_fn); self.attributes @@ -261,10 +263,10 @@ unsafe impl crate::object::Traverse for PyType { if let Some(base) = unsafe { self.base.swap(None) } { out.push(base.into()); } - if let Some(mut guard) = self.bases.try_write() { - for base in guard.drain(..) { - out.push(base.into()); - } + if let Some(mut bases) = self.bases.try_write() { + let empty = object::PyBaseObject::static_type().bases.read().clone(); + let old_bases = core::mem::replace(&mut *bases, empty); + out.push(old_bases.into_untyped().into()); } if let Some(mut guard) = self.mro.try_write() { for typ in guard.drain(..) { @@ -590,6 +592,7 @@ impl PyType { type_data: PyRwLock::new(None), specialization_cache: TypeSpecializationCache::new(), }; + let bases = PyTuple::new_ref_typed(bases, ctx); let base = bases[0].clone(); Self::new_heap_inner(base, bases, attrs, slots, heaptype_ext, metaclass, ctx) @@ -758,7 +761,7 @@ impl PyType { #[allow(clippy::too_many_arguments)] fn new_heap_inner( base: PyRef, - bases: Vec>, + bases: PyTypeTupleRef, attrs: PyAttributes, mut slots: PyTypeSlots, heaptype_ext: HeapTypeExt, @@ -872,13 +875,14 @@ impl PyType { } let inherited_abc_tpflags = Self::inherited_abc_tpflags(core::slice::from_ref(&base)); - let bases = PyRwLock::new(vec![base.clone()]); + let bases = + PyTuple::new_ref_typed_with_type(vec![base.clone()], PyTuple::static_type().to_owned()); let mro = base.mro_map_collect(|x| x.to_owned()); let new_type = PyRef::new_ref( Self { base: Some(base).into(), - bases, + bases: PyRwLock::new(bases), mro: PyRwLock::new(mro), subclasses: PyRwLock::default(), attributes: PyRwLock::new(attrs), @@ -991,6 +995,11 @@ impl PyType { } } + pub(crate) fn finalize_bootstrap_static(typ: &Py) { + Self::set_new(&typ.slots, typ.base.deref()); + Self::set_alloc(&typ.slots, typ.base.deref()); + } + /// Inherit readonly slots from base type at creation time. /// These slots are not AtomicCell and must be set before the type is used. fn inherit_readonly_slots(slots: &mut PyTypeSlots, base: &Self) { @@ -1466,16 +1475,10 @@ impl Py { impl PyType { #[pygetset] fn __bases__(&self, vm: &VirtualMachine) -> PyTupleRef { - vm.ctx.new_tuple( - self.bases - .read() - .iter() - .map(|x| x.as_object().to_owned()) - .collect(), - ) + Self::with_type_lock(vm, || self.bases.read().clone().into_untyped()) } #[pygetset(setter, name = "__bases__")] - fn set_bases(zelf: &Py, bases: Vec, vm: &VirtualMachine) -> PyResult<()> { + fn set_bases(zelf: &Py, bases_tuple: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { // TODO: Assigning to __bases__ is only used in typing.NamedTupleMeta.__new__ // Rather than correctly re-initializing the class, we are skipping a few steps for now if zelf.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { @@ -1484,12 +1487,22 @@ impl PyType { zelf.name() ))); } - if bases.is_empty() { + if bases_tuple.is_empty() { return Err(vm.new_type_error(format!( "can only assign non-empty tuple to {}.__bases__, not ()", zelf.name() ))); } + for base in bases_tuple.iter() { + if base.downcast_ref::().is_none() { + return Err(vm.new_type_error(format!( + "{}.__bases__ must be tuple of classes, not '{}'", + zelf.name(), + base.class().name() + ))); + } + } + let bases = bases_tuple.try_into_typed::(vm)?; // TODO: check for mro cycles @@ -1595,7 +1608,7 @@ impl PyType { keep_alive(failed_base, &mut retired); } register_subclasses(&zelf.bases.read()); - retired.extend(failed_bases.into_iter().map(Into::into)); + retired.push(failed_bases.into_untyped().into()); zelf.modified_inner(); return Err(err); } @@ -1605,7 +1618,7 @@ impl PyType { retired.extend(old_mro.into_iter().map(Into::into)); retired.push(cls.into()); } - retired.extend(old_bases.into_iter().map(Into::into)); + retired.push(old_bases.into_untyped().into()); if let Some(old_base) = old_base { keep_alive(old_base, &mut retired); } @@ -2107,26 +2120,24 @@ impl Constructor for PyType { let (metatype, base, bases, base_is_type) = if bases.is_empty() { let base = vm.ctx.types.object_type.to_owned(); - (metatype, base.clone(), vec![base], false) + let bases = PyTuple::new_ref_typed(vec![base.clone()], &vm.ctx); + (metatype, base, bases, false) } else { - let bases = bases - .iter() - .map(|obj| { - obj.clone().downcast::().or_else(|obj| { - if vm - .get_attribute_opt(obj, identifier!(vm, __mro_entries__))? - .is_some() - { - Err(vm.new_type_error( - "type() doesn't support MRO entry resolution; \ - use types.new_class()", - )) - } else { - Err(vm.new_type_error("bases must be types")) - } - }) - }) - .collect::>>()?; + for obj in bases.iter() { + if obj.downcast_ref::().is_none() { + if vm + .get_attribute_opt(obj.clone(), identifier!(vm, __mro_entries__))? + .is_some() + { + return Err(vm.new_type_error( + "type() doesn't support MRO entry resolution; \ + use types.new_class()", + )); + } + return Err(vm.new_type_error("bases must be types")); + } + } + let bases = bases.try_into_typed::(vm)?; // Search the bases for the proper metatype to deal with this: let winner = calculate_meta_class(metatype.clone(), &bases, vm)?; diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 228fe1290ea..f9affd499b6 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -17,7 +17,7 @@ use super::{ }; use crate::object::traverse_object::PyObjVTable; use crate::{ - builtins::{PyDictRef, PyType, PyTypeRef}, + builtins::{PyDictRef, PyTuple, PyTupleRef, PyType, PyTypeRef, type_::PyTypeTupleRef}, common::{ atomic::{Ordering, PyAtomic, Radium}, linked_list::{Link, Pointers}, @@ -2309,12 +2309,12 @@ impl Clone for PyRef { } impl PyRef { - // #[inline(always)] - // pub(crate) const fn into_non_null(self) -> NonNull> { - // let ptr = self.ptr; - // std::mem::forget(self); - // ptr - // } + #[inline(always)] + pub(super) const fn into_non_null(self) -> NonNull> { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } #[inline(always)] pub(crate) const unsafe fn from_non_null(ptr: NonNull>) -> Self { @@ -2547,155 +2547,195 @@ impl PyWeakRef { /// Partially initialize a struct, ensuring that all fields are /// either given values or explicitly left uninitialized -macro_rules! partially_init { - ( - $ty:path {$($init_field:ident: $init_value:expr),*$(,)?}, - Uninit { $($uninit_field:ident),*$(,)? }$(,)? - ) => {{ - // check all the fields are there but *don't* actually run it - - #[allow(clippy::diverging_sub_expression, reason = "intentional compile-time field check in an unreachable branch")] - if false { - #[allow(invalid_value, dead_code, unreachable_code)] - let _ = {$ty { - $($init_field: $init_value,)* - $($uninit_field: unreachable!(),)* - }}; - } - let mut m = ::core::mem::MaybeUninit::<$ty>::uninit(); - #[allow(unused_unsafe)] - unsafe { - $(::core::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* - } - m - }}; -} - -pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { - use crate::{builtins::object, class::PyClassImpl}; +pub(crate) struct BootstrapTypeHierarchy { + pub type_type: PyTypeRef, + pub object_type: PyTypeRef, + pub tuple_type: PyTypeRef, + pub weakref_type: PyTypeRef, + pub empty_tuple: PyTupleRef, +} + +pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { + use crate::{ + builtins::{object, tuple}, + class::PyClassImpl, + }; use core::mem::MaybeUninit; - // `type` inherits from `object` - // and both `type` and `object are instances of `type`. - // to produce this circular dependency, we need an unsafe block. - // (and yes, this will never get dropped. TODO?) - let (type_type, object_type) = { - // We cast between these 2 types, so make sure (at compile time) that there's no change in - // layout when we wrap PyInner in MaybeUninit<> - static_assertions::assert_eq_size!(MaybeUninit>, PyInner); - static_assertions::assert_eq_align!(MaybeUninit>, PyInner); - - let type_payload = PyType { - base: None.into(), - bases: PyRwLock::default(), - mro: PyRwLock::default(), - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(Default::default()), - slots: PyType::make_slots(), - heaptype_ext: None, - tp_version_tag: core::sync::atomic::AtomicU32::new(0), - abc_tpflags: core::sync::atomic::AtomicU64::new(0), - }; - let object_payload = PyType { - base: None.into(), - bases: PyRwLock::default(), - mro: PyRwLock::default(), - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(Default::default()), - slots: object::PyBaseObject::make_slots(), - heaptype_ext: None, - tp_version_tag: core::sync::atomic::AtomicU32::new(0), - abc_tpflags: core::sync::atomic::AtomicU64::new(0), - }; - // Both type_type and object_type are instances of `type`, which has - // HAS_DICT and HAS_WEAKREF, so they need both ObjExt and WeakRefList prefixes. - // Layout: [ObjExt][WeakRefList][PyInner] - let alloc_type_with_prefixes = || -> *mut MaybeUninit> { - let inner_layout = core::alloc::Layout::new::>>(); - let ext_layout = core::alloc::Layout::new::(); - let weakref_layout = core::alloc::Layout::new::(); - - let (layout, weakref_offset) = ext_layout.extend(weakref_layout).unwrap(); - let (combined, inner_offset) = layout.extend(inner_layout).unwrap(); - let combined = combined.pad_to_align(); - - let alloc_ptr = unsafe { alloc::alloc::alloc(combined) }; - if alloc_ptr.is_null() { - alloc::alloc::handle_alloc_error(combined); - } - alloc_ptr.expose_provenance(); + static_assertions::assert_eq_size!(MaybeUninit>, PyInner); + static_assertions::assert_eq_align!(MaybeUninit>, PyInner); + static_assertions::assert_eq_size!(MaybeUninit>, PyInner); + static_assertions::assert_eq_align!(MaybeUninit>, PyInner); - unsafe { - let ext_ptr = alloc_ptr as *mut ObjExt; - ext_ptr.write(ObjExt::new(None, 0, true)); + // All three core type objects are instances of `type`, which has HAS_DICT + // and HAS_WEAKREF. Their allocations therefore need both prefixes. + let alloc_type_with_prefixes = || -> *mut PyInner { + let inner_layout = core::alloc::Layout::new::>>(); + let ext_layout = core::alloc::Layout::new::(); + let weakref_layout = core::alloc::Layout::new::(); - let weakref_ptr = alloc_ptr.add(weakref_offset) as *mut WeakRefList; - weakref_ptr.write(WeakRefList::new()); + let (layout, weakref_offset) = ext_layout.extend(weakref_layout).unwrap(); + let (combined, inner_offset) = layout.extend(inner_layout).unwrap(); + let combined = combined.pad_to_align(); - alloc_ptr.add(inner_offset) as *mut MaybeUninit> - } - }; - - let type_type_ptr = alloc_type_with_prefixes(); - unsafe { - type_type_ptr.write(partially_init!( - PyInner:: { - ref_count: RefCount::new(), - vtable: PyObjVTable::of::(), - gc_bits: Radium::new(0), - gc_generation: Radium::new(GC_UNTRACKED), - gc_pointers: Pointers::new(), - payload: type_payload, - }, - Uninit { typ } - )); + let alloc_ptr = unsafe { alloc::alloc::alloc(combined) }; + if alloc_ptr.is_null() { + alloc::alloc::handle_alloc_error(combined); } + alloc_ptr.expose_provenance(); - let object_type_ptr = alloc_type_with_prefixes(); unsafe { - object_type_ptr.write(partially_init!( - PyInner:: { - ref_count: RefCount::new(), - vtable: PyObjVTable::of::(), - gc_bits: Radium::new(0), - gc_generation: Radium::new(GC_UNTRACKED), - gc_pointers: Pointers::new(), - payload: object_payload, - }, - Uninit { typ }, - )); + (alloc_ptr as *mut ObjExt).write(ObjExt::new(None, 0, true)); + (alloc_ptr.add(weakref_offset) as *mut WeakRefList).write(WeakRefList::new()); + alloc_ptr.add(inner_offset).cast() } + }; - let object_type_ptr = object_type_ptr as *mut PyInner; - let type_type_ptr = type_type_ptr as *mut PyInner; + let alloc_tuple = || { + Box::into_raw(Box::new(MaybeUninit::>::uninit())) + .cast::>() + }; - unsafe { - (*type_type_ptr).ref_count.inc(); - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - ptr::write(&mut (*object_type_ptr).typ, PyAtomicRef::from(type_type)); - (*type_type_ptr).ref_count.inc(); - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - ptr::write(&mut (*type_type_ptr).typ, PyAtomicRef::from(type_type)); + unsafe fn init_ref_count(ptr: *mut PyInner) { + unsafe { ptr::addr_of_mut!((*ptr).ref_count).write(RefCount::new()) }; + } - let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); - // object's mro is [object] - (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); + unsafe fn initial_ref(ptr: *mut PyInner) -> PyRef { + unsafe { PyRef::from_raw(ptr.cast()) } + } - (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); - (*type_type_ptr).payload.base = Some(object_type.clone()).into(); + unsafe fn clone_raw_ref(ptr: *mut PyInner) -> PyRef { + unsafe { &*ptr::addr_of!((*ptr).ref_count) }.inc(); + unsafe { PyRef::from_raw(ptr.cast()) } + } - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - // type's mro is [type, object] - (*type_type_ptr).payload.mro = - PyRwLock::new(vec![type_type.clone(), object_type.clone()]); + unsafe fn into_type_tuple(tuple: PyTupleRef) -> PyTypeTupleRef { + // SAFETY: PyTypeRef and PyObjectRef have the same layout, and the + // bootstrap tuples contain only PyType objects. + unsafe { core::mem::transmute::(tuple) } + } - (type_type, object_type) + unsafe fn init_inner(ptr: *mut PyInner, typ: PyTypeRef, payload: T) + where + T: PyPayload + MaybeTraverse + fmt::Debug, + { + unsafe { + ptr::addr_of_mut!((*ptr).vtable).write(PyObjVTable::of::()); + ptr::addr_of_mut!((*ptr).gc_bits).write(Radium::new(0)); + ptr::addr_of_mut!((*ptr).gc_generation).write(Radium::new(GC_UNTRACKED)); + ptr::addr_of_mut!((*ptr).gc_pointers).write(Pointers::new()); + ptr::addr_of_mut!((*ptr).typ).write(PyAtomicRef::from_ref_without_retag(typ)); + ptr::addr_of_mut!((*ptr).payload).write(payload); } + } + + let type_type_ptr = alloc_type_with_prefixes(); + let object_type_ptr = alloc_type_with_prefixes(); + let tuple_type_ptr = alloc_type_with_prefixes(); + let empty_tuple_ptr = alloc_tuple(); + let type_bases_ptr = alloc_tuple(); + let tuple_bases_ptr = alloc_tuple(); + + unsafe { + init_ref_count(type_type_ptr); + init_ref_count(object_type_ptr); + init_ref_count(tuple_type_ptr); + init_ref_count(empty_tuple_ptr); + init_ref_count(type_bases_ptr); + init_ref_count(tuple_bases_ptr); + } + + // Each initial reference consumes the allocation's initial strong count. + // Further references are created through clone_raw_ref while the graph is + // still being assembled and cannot yet be safely dereferenced. + let type_type = unsafe { initial_ref(type_type_ptr) }; + let object_type = unsafe { initial_ref(object_type_ptr) }; + let tuple_type = unsafe { initial_ref(tuple_type_ptr) }; + let empty_tuple = unsafe { initial_ref(empty_tuple_ptr) }; + let type_bases = unsafe { into_type_tuple(initial_ref(type_bases_ptr)) }; + let tuple_bases = unsafe { into_type_tuple(initial_ref(tuple_bases_ptr)) }; + + let type_payload = PyType { + base: unsafe { + PyAtomicRef::from_optional_ref_without_retag(Some(clone_raw_ref(object_type_ptr))) + }, + bases: PyRwLock::new(type_bases), + mro: PyRwLock::new(vec![unsafe { clone_raw_ref(type_type_ptr) }, unsafe { + clone_raw_ref(object_type_ptr) + }]), + subclasses: PyRwLock::default(), + attributes: PyRwLock::default(), + slots: PyType::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + abc_tpflags: core::sync::atomic::AtomicU64::new(0), }; + let object_payload = PyType { + base: unsafe { PyAtomicRef::from_optional_ref_without_retag(None) }, + bases: PyRwLock::new(unsafe { into_type_tuple(clone_raw_ref(empty_tuple_ptr)) }), + mro: PyRwLock::new(vec![unsafe { clone_raw_ref(object_type_ptr) }]), + subclasses: PyRwLock::default(), + attributes: PyRwLock::default(), + slots: object::PyBaseObject::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + abc_tpflags: core::sync::atomic::AtomicU64::new(0), + }; + let tuple_payload = PyType { + base: unsafe { + PyAtomicRef::from_optional_ref_without_retag(Some(clone_raw_ref(object_type_ptr))) + }, + bases: PyRwLock::new(tuple_bases), + mro: PyRwLock::new(vec![unsafe { clone_raw_ref(tuple_type_ptr) }, unsafe { + clone_raw_ref(object_type_ptr) + }]), + subclasses: PyRwLock::default(), + attributes: PyRwLock::default(), + slots: tuple::PyTuple::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + abc_tpflags: core::sync::atomic::AtomicU64::new(0), + }; + + let object_element = + || -> PyObjectRef { unsafe { clone_raw_ref::(object_type_ptr) }.into() }; + unsafe { + init_inner(type_type_ptr, clone_raw_ref(type_type_ptr), type_payload); + init_inner( + object_type_ptr, + clone_raw_ref(type_type_ptr), + object_payload, + ); + init_inner(tuple_type_ptr, clone_raw_ref(type_type_ptr), tuple_payload); + init_inner( + empty_tuple_ptr, + clone_raw_ref(tuple_type_ptr), + PyTuple::new_unchecked(Vec::new().into_boxed_slice()), + ); + init_inner( + type_bases_ptr, + clone_raw_ref(tuple_type_ptr), + PyTuple::new_unchecked(vec![object_element()].into_boxed_slice()), + ); + init_inner( + tuple_bases_ptr, + clone_raw_ref(tuple_type_ptr), + PyTuple::new_unchecked(vec![object_element()].into_boxed_slice()), + ); + } - let weakref_type = PyType { + PyType::finalize_bootstrap_static(&tuple_type); + + let weakref_bases = + PyTuple::new_ref_typed_with_type(vec![object_type.clone()], tuple_type.clone()); + unsafe { + crate::gc_state::gc_state() + .untrack_object(NonNull::from(weakref_bases.as_untyped().as_object())); + } + weakref_bases.as_untyped().as_object().clear_gc_tracked(); + let weakref_payload = PyType { base: Some(object_type.clone()).into(), - bases: PyRwLock::new(vec![object_type.clone()]), + bases: PyRwLock::new(weakref_bases), mro: PyRwLock::new(vec![object_type.clone()]), subclasses: PyRwLock::default(), attributes: PyRwLock::default(), @@ -2704,7 +2744,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { tp_version_tag: core::sync::atomic::AtomicU32::new(0), abc_tpflags: core::sync::atomic::AtomicU64::new(0), }; - let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); + let weakref_type = PyRef::new_ref(weakref_payload, type_type.clone(), None); // Static type: untrack from GC (was tracked by new_ref because PyType has HAS_TRAVERSE) unsafe { crate::gc_state::gc_state() @@ -2721,6 +2761,13 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { .unwrap(), ); + object_type.subclasses.write().push( + tuple_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); + object_type.subclasses.write().push( weakref_type .as_object() @@ -2728,7 +2775,13 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { .unwrap(), ); - (type_type, object_type, weakref_type) + BootstrapTypeHierarchy { + type_type, + object_type, + tuple_type, + weakref_type, + empty_tuple, + } } #[cfg(test)] @@ -2737,7 +2790,29 @@ mod tests { #[test] fn miri_test_type_initialization() { - let _ = init_type_hierarchy(); + let hierarchy = init_type_hierarchy(); + + assert!(hierarchy.type_type.class().is(&hierarchy.type_type)); + assert!(hierarchy.object_type.class().is(&hierarchy.type_type)); + assert!(hierarchy.tuple_type.class().is(&hierarchy.type_type)); + assert!(hierarchy.weakref_type.class().is(&hierarchy.type_type)); + + let object_bases = hierarchy.object_type.bases.read(); + assert!(object_bases.is_empty()); + assert!(object_bases.as_untyped().is(&hierarchy.empty_tuple)); + assert!(object_bases.as_untyped().class().is(&hierarchy.tuple_type)); + drop(object_bases); + + for typ in [ + &hierarchy.type_type, + &hierarchy.tuple_type, + &hierarchy.weakref_type, + ] { + let bases = typ.bases.read(); + assert_eq!(bases.len(), 1); + assert!(bases[0].is(&hierarchy.object_type)); + assert!(bases.as_untyped().class().is(&hierarchy.tuple_type)); + } } #[test] diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index e576ac2c191..53caad289cc 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -312,6 +312,22 @@ impl Deref for PyAtomicRef { } impl PyAtomicRef { + /// Move a reference into an atomic pointer without creating a Rust + /// reference to the pointee. This is only for bootstrap objects whose + /// allocation is valid but whose payload is still being initialized. + /// + /// # Safety + /// The pointee must remain allocated, and this atomic reference must not + /// be dereferenced until the pointee has been fully initialized. + pub(super) unsafe fn from_ref_without_retag(pyref: PyRef) -> Self { + let ptr = pyref.into_non_null().as_ptr().cast::(); + ptr.expose_provenance(); + Self { + inner: Radium::new(ptr), + _phantom: Default::default(), + } + } + /// Load the raw pointer without creating a reference. /// Avoids Stacked Borrows retag, safe for use during bootstrap /// when type objects have self-referential pointers being mutated. @@ -349,6 +365,22 @@ impl From>> for PyAtomicRef> { } impl PyAtomicRef> { + /// Optional form of PyAtomicRef::from_ref_without_retag. + /// + /// # Safety + /// A non-None pointee must remain allocated, and this atomic reference + /// must not be dereferenced until the pointee has been fully initialized. + pub(super) unsafe fn from_optional_ref_without_retag(opt_ref: Option>) -> Self { + let ptr = opt_ref.map_or(null_mut(), |pyref| { + pyref.into_non_null().as_ptr().cast::() + }); + ptr.expose_provenance(); + Self { + inner: Radium::new(ptr), + _phantom: Default::default(), + } + } + pub fn deref(&self) -> Option<&Py> { self.deref_ordering(Ordering::Relaxed) } diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 64807fc0973..1f2ebaab65d 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -109,18 +109,19 @@ pub struct TypeZoo { impl TypeZoo { #[cold] - pub(crate) fn init() -> Self { - let (type_type, object_type, weakref_type) = crate::object::init_type_hierarchy(); - // the order matters for type, object, weakref, and int - must be initialized first - let type_type = type_::PyType::init_manually(type_type); - let object_type = object::PyBaseObject::init_manually(object_type); - let weakref_type = weakref::PyWeak::init_manually(weakref_type); + pub(crate) fn init() -> (Self, crate::builtins::PyTupleRef) { + let hierarchy = crate::object::init_type_hierarchy(); + // These core types must be published before any other static type is created. + let type_type = type_::PyType::init_manually(hierarchy.type_type); + let object_type = object::PyBaseObject::init_manually(hierarchy.object_type); + let tuple_type = tuple::PyTuple::init_manually(hierarchy.tuple_type); + let weakref_type = weakref::PyWeak::init_manually(hierarchy.weakref_type); let int_type = int::PyInt::init_builtin_type(); // builtin_function_or_method and builtin_method share the same type (CPython behavior) let builtin_function_or_method_type = builtin_func::PyNativeFunction::init_builtin_type(); - Self { + let types = Self { type_type, object_type, weakref_type, @@ -147,7 +148,7 @@ impl TypeZoo { staticmethod_type: staticmethod::PyStaticMethod::init_builtin_type(), str_type: pystr::PyStr::init_builtin_type(), super_type: super_::PySuper::init_builtin_type(), - tuple_type: tuple::PyTuple::init_builtin_type(), + tuple_type, zip_type: zip::PyZip::init_builtin_type(), // hidden internal types. is this really need to be cached here? @@ -213,7 +214,8 @@ impl TypeZoo { method_wrapper_type: descriptor::PyMethodWrapper::init_builtin_type(), method_def: crate::function::HeapMethodDef::init_builtin_type(), - } + }; + (types, hierarchy.empty_tuple) } /// Fill attributes of builtin types. diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 9a545663576..615af24e63d 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -304,7 +304,7 @@ impl Context { fn init_genesis() -> Self { flame_guard!("init Context"); - let types = TypeZoo::init(); + let (types, empty_tuple) = TypeZoo::init(); let exceptions = exceptions::ExceptionZoo::init(); #[inline] @@ -340,10 +340,6 @@ impl Context { let true_value = create_object(PyBool(PyInt::from(1)), types.bool_type); let false_value = create_object(PyBool(PyInt::from(0)), types.bool_type); - let empty_tuple = create_object( - PyTuple::new_unchecked(Vec::new().into_boxed_slice()), - types.tuple_type, - ); let empty_frozenset = PyRef::new_ref( PyFrozenSet::default(), types.frozenset_type.to_owned(),