From 8d4964319d49a99433bb0b70a5cef43bbd5e4515 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:15:35 +0100 Subject: [PATCH 01/77] release: 0.17.0 --- CHANGELOG.md | 4 +++ Cargo.toml | 6 ++-- src/de.rs | 60 +++++++++++++++++++------------------- tests/test_custom_types.rs | 4 ++- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d58d8..27181f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.17.0 - 2022-08-24 + +- Update to PyO3 0.17 + ## 0.16.0 - 2022-03-06 - Update to PyO3 0.16 diff --git a/Cargo.toml b/Cargo.toml index 3ff2974..f80e26d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.16.0" +version = "0.17.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2018" license = "MIT" @@ -12,10 +12,10 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.16.3", default-features = false } +pyo3 = { version = "0.17.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.16.3", default-features = false, features = ["auto-initialize", "macros", "pyproto"] } +pyo3 = { version = "0.17.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" diff --git a/src/de.rs b/src/de.rs index 1f4253d..197ca3e 100644 --- a/src/de.rs +++ b/src/de.rs @@ -433,16 +433,16 @@ mod test { where T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - let actual: T = depythonize(obj).unwrap(); - assert_eq!(&actual, expected); - let actual_json: JsonValue = depythonize(obj).unwrap(); - assert_eq!(&actual_json, expected_json); + Python::with_gil(|py| { + let locals = PyDict::new(py); + py.run(&format!("obj = {}", code), None, Some(locals)) + .unwrap(); + let obj = locals.get_item("obj").unwrap(); + let actual: T = depythonize(obj).unwrap(); + assert_eq!(&actual, expected); + let actual_json: JsonValue = depythonize(obj).unwrap(); + assert_eq!(&actual_json, expected_json); + }); } #[test] @@ -489,16 +489,16 @@ mod test { let code = "{'foo': 'Foo'}"; - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - assert!(matches!( - *depythonize::(obj).unwrap_err().inner, - ErrorImpl::Message(msg) if msg == "missing field `bar`" - )); + Python::with_gil(|py| { + let locals = PyDict::new(py); + py.run(&format!("obj = {}", code), None, Some(locals)) + .unwrap(); + let obj = locals.get_item("obj").unwrap(); + assert!(matches!( + *depythonize::(obj).unwrap_err().inner, + ErrorImpl::Message(msg) if msg == "missing field `bar`" + )); + }) } #[test] @@ -519,16 +519,16 @@ mod test { let code = "('cat', -10.05, 'foo')"; - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - assert!(matches!( - *depythonize::(obj).unwrap_err().inner, - ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 - )); + Python::with_gil(|py| { + let locals = PyDict::new(py); + py.run(&format!("obj = {}", code), None, Some(locals)) + .unwrap(); + let obj = locals.get_item("obj").unwrap(); + assert!(matches!( + *depythonize::(obj).unwrap_err().inner, + ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 + )); + }) } #[test] diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index ff437cc..fd19417 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -10,7 +10,7 @@ use pythonize::{ }; use serde_json::{json, Value}; -#[pyclass] +#[pyclass(sequence)] struct CustomList { items: Vec, } @@ -62,6 +62,7 @@ impl PythonizeTypes for PythonizeCustomList { #[test] fn test_custom_list() { Python::with_gil(|py| { + PySequence::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!([1, 2, 3])) .unwrap() .into_ref(py); @@ -125,6 +126,7 @@ impl PythonizeTypes for PythonizeCustomDict { #[test] fn test_custom_dict() { Python::with_gil(|py| { + PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) .unwrap() From cfce252c3b5c969503b332107352fddee7d34653 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 27 Dec 2022 13:14:34 +0000 Subject: [PATCH 02/77] add LICENSE --- LICENSE | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ace026e --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2022-present David Hewitt and Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 4dbb21174457f672162bab40bbe37586b6aaea05 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 22 Jan 2023 07:54:12 +0000 Subject: [PATCH 03/77] release: 0.18.0 --- CHANGELOG.md | 5 +++++ Cargo.toml | 6 +++--- src/de.rs | 14 +++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27181f8..fe8a050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.18.0 - 2022-08-24 + +- Add LICENSE file to the crate +- Update to PyO3 0.18 + ## 0.17.0 - 2022-08-24 - Update to PyO3 0.17 diff --git a/Cargo.toml b/Cargo.toml index f80e26d..97816a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.17.0" +version = "0.18.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2018" license = "MIT" @@ -12,10 +12,10 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.17.0", default-features = false } +pyo3 = { version = "0.18.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.17.0", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.18.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" diff --git a/src/de.rs b/src/de.rs index 197ca3e..c6a171f 100644 --- a/src/de.rs +++ b/src/de.rs @@ -107,7 +107,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s = self.input.cast_as::()?.to_str()?; + let s = self.input.downcast::()?.to_str()?; if s.len() != 1 { return Err(PythonizeError::invalid_length_char()); } @@ -129,7 +129,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s: &PyString = self.input.cast_as()?; + let s: &PyString = self.input.downcast()?; visitor.visit_str(s.to_str()?) } @@ -145,7 +145,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { V: de::Visitor<'de>, { let obj = self.input; - let b: &PyBytes = obj.cast_as()?; + let b: &PyBytes = obj.downcast()?; visitor.visit_bytes(b.as_bytes()) } @@ -249,20 +249,20 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { let item = self.input; if item.is_instance_of::()? { // Get the enum variant from the dict key - let d: &PyDict = item.cast_as().unwrap(); + let d: &PyDict = item.downcast().unwrap(); if d.len() != 1 { return Err(PythonizeError::invalid_length_enum()); } let variant: &PyString = d .keys() .get_item(0)? - .cast_as() + .downcast() .map_err(|_| PythonizeError::dict_key_not_string())?; let value = d.get_item(variant).unwrap(); let mut de = Depythonizer::from_object(value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) } else if item.is_instance_of::()? { - let s: &PyString = self.input.cast_as()?; + let s: &PyString = self.input.downcast()?; visitor.visit_enum(s.to_str()?.into_deserializer()) } else { Err(PythonizeError::invalid_enum_type()) @@ -275,7 +275,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { { let s: &PyString = self .input - .cast_as() + .downcast() .map_err(|_| PythonizeError::dict_key_not_string())?; visitor.visit_str(s.to_str()?) } From 63e573a3af2d56ccbe7699d874f1af0ed2af51cf Mon Sep 17 00:00:00 2001 From: Zak Stucke Date: Mon, 5 Jun 2023 14:55:44 +0100 Subject: [PATCH 04/77] Bumped Pyo3 to 0.19 --- CHANGELOG.md | 4 ++++ Cargo.toml | 4 ++-- src/de.rs | 28 ++++++++++++++-------------- src/lib.rs | 24 ++++++++++++------------ tests/test_custom_types.rs | 4 ++-- 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8a050..6a9fad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +- Update to PyO3 0.19 + ## 0.18.0 - 2022-08-24 - Add LICENSE file to the crate diff --git a/Cargo.toml b/Cargo.toml index 97816a1..c0e06b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,10 +12,10 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.18.0", default-features = false } +pyo3 = { version = "0.19.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.18.0", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.19.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" diff --git a/src/de.rs b/src/de.rs index c6a171f..3e1b097 100644 --- a/src/de.rs +++ b/src/de.rs @@ -61,29 +61,29 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { if obj.is_none() { self.deserialize_unit(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if obj.is_instance_of::()? || obj.is_instance_of::()? { + } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_bytes(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_map(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_f64(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_i64(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_i64(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_str(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_str(visitor) } else if let Ok(_) = obj.downcast::() { self.deserialize_tuple(obj.len()?, visitor) @@ -247,7 +247,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { V: de::Visitor<'de>, { let item = self.input; - if item.is_instance_of::()? { + if item.is_instance_of::() { // Get the enum variant from the dict key let d: &PyDict = item.downcast().unwrap(); if d.len() != 1 { @@ -261,7 +261,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { let value = d.get_item(variant).unwrap(); let mut de = Depythonizer::from_object(value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) - } else if item.is_instance_of::()? { + } else if item.is_instance_of::() { let s: &PyString = self.input.downcast()?; visitor.visit_enum(s.to_str()?.into_deserializer()) } else { diff --git a/src/lib.rs b/src/lib.rs index 139f684..3e941b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,23 +18,23 @@ //! bar: Option //! } //! -//! let gil = Python::acquire_gil(); -//! let py = gil.python(); +//! Python::with_gil(|py| { +//! let sample = Sample { +//! foo: "Foo".to_string(), +//! bar: None +//! }; //! -//! let sample = Sample { -//! foo: "Foo".to_string(), -//! bar: None -//! }; +//! // Rust -> Python +//! let obj = pythonize(py, &sample).unwrap(); //! -//! // Rust -> Python -//! let obj = pythonize(py, &sample).unwrap(); +//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); //! -//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); +//! // Python -> Rust +//! let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); //! -//! // Python -> Rust -//! let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); +//! assert_eq!(new_sample, sample); +//! }); //! -//! assert_eq!(new_sample, sample); //! ``` mod de; mod error; diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index fd19417..e99bcbe 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -66,7 +66,7 @@ fn test_custom_list() { let serialized = pythonize_custom::(py, &json!([1, 2, 3])) .unwrap() .into_ref(py); - assert!(serialized.is_instance_of::().unwrap()); + assert!(serialized.is_instance_of::()); let deserialized: Value = depythonize(serialized).unwrap(); assert_eq!(deserialized, json!([1, 2, 3])); @@ -131,7 +131,7 @@ fn test_custom_dict() { pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) .unwrap() .into_ref(py); - assert!(serialized.is_instance_of::().unwrap()); + assert!(serialized.is_instance_of::()); let deserialized: Value = depythonize(serialized).unwrap(); assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); From 64bb7ba98de62fc974ebe60f42c45e257fa44a62 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 11 Jun 2023 21:22:02 +0100 Subject: [PATCH 05/77] release: 0.19 --- CHANGELOG.md | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9fad0..7d69d1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ -## Unreleased +## 0.19.0 - 2023-06-11 - Update to PyO3 0.19 -## 0.18.0 - 2022-08-24 +## 0.18.0 - 2023-01-22 - Add LICENSE file to the crate - Update to PyO3 0.18 diff --git a/Cargo.toml b/Cargo.toml index c0e06b1..ff5dc2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.18.0" +version = "0.19.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2018" license = "MIT" From 3d8678a7583b42a996905ca1244d73ce432858aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Berson?= Date: Sat, 14 Oct 2023 10:36:53 +0200 Subject: [PATCH 06/77] Bumped Pyo3 to 0.20 --- CHANGELOG.md | 4 ++++ Cargo.toml | 4 ++-- src/de.rs | 10 +++++----- src/ser.rs | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d69d1c..102e343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +- Update to PyO3 0.20 + ## 0.19.0 - 2023-06-11 - Update to PyO3 0.19 diff --git a/Cargo.toml b/Cargo.toml index ff5dc2c..7f7ac44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,10 +12,10 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.19.0", default-features = false } +pyo3 = { version = "0.20.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.19.0", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.20.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" diff --git a/src/de.rs b/src/de.rs index 3e1b097..3cd181b 100644 --- a/src/de.rs +++ b/src/de.rs @@ -85,7 +85,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { self.deserialize_tuple(obj.len()?, visitor) } else if obj.is_instance_of::() { self.deserialize_str(visitor) - } else if let Ok(_) = obj.downcast::() { + } else if obj.downcast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.downcast::().is_ok() { self.deserialize_map(visitor) @@ -258,7 +258,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { .get_item(0)? .downcast() .map_err(|_| PythonizeError::dict_key_not_string())?; - let value = d.get_item(variant).unwrap(); + let value = d.get_item(variant)?.unwrap(); let mut de = Depythonizer::from_object(value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) } else if item.is_instance_of::() { @@ -437,7 +437,7 @@ mod test { let locals = PyDict::new(py); py.run(&format!("obj = {}", code), None, Some(locals)) .unwrap(); - let obj = locals.get_item("obj").unwrap(); + let obj = locals.get_item("obj").unwrap().unwrap(); let actual: T = depythonize(obj).unwrap(); assert_eq!(&actual, expected); let actual_json: JsonValue = depythonize(obj).unwrap(); @@ -493,7 +493,7 @@ mod test { let locals = PyDict::new(py); py.run(&format!("obj = {}", code), None, Some(locals)) .unwrap(); - let obj = locals.get_item("obj").unwrap(); + let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( *depythonize::(obj).unwrap_err().inner, ErrorImpl::Message(msg) if msg == "missing field `bar`" @@ -523,7 +523,7 @@ mod test { let locals = PyDict::new(py); py.run(&format!("obj = {}", code), None, Some(locals)) .unwrap(); - let obj = locals.get_item("obj").unwrap(); + let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( *depythonize::(obj).unwrap_err().inner, ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 diff --git a/src/ser.rs b/src/ser.rs index a1f0d86..794498b 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -474,7 +474,7 @@ mod test { None, Some(locals), )?; - let result = locals.get_item("result").unwrap().extract::<&str>()?; + let result = locals.get_item("result")?.unwrap().extract::<&str>()?; assert_eq!(result, expected); assert_eq!(serde_json::to_string(&src).unwrap(), expected); From ceb2f86f82050735e1a566bfd2830585692bc24d Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:31:34 +0100 Subject: [PATCH 07/77] release: 0.20 --- CHANGELOG.md | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 102e343..dab69ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.20.0 - 2023-10-15 - Update to PyO3 0.20 diff --git a/Cargo.toml b/Cargo.toml index 7f7ac44..bbadfdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.19.0" +version = "0.20.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2018" license = "MIT" From 4a214e47022a19b0c01ab060efba7b87f8e066ba Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:36:12 +0100 Subject: [PATCH 08/77] update versions in CI --- .github/workflows/ci.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7133743..ddbfdf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,17 +35,16 @@ jobs: build: needs: [fmt] # don't wait for clippy as fails rarely and takes longer - name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} ${{ matrix.msrv }} + name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} runs-on: ${{ matrix.platform.os }} strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] platform: [ - { os: "macOS-latest", python-architecture: "x64", rust-target: "x86_64-apple-darwin" }, - { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" }, - { os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" }, - { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" }, + { os: "macOS-latest", rust-target: "x86_64-apple-darwin" }, + { os: "ubuntu-latest", rust-target: "x86_64-unknown-linux-gnu" }, + { os: "windows-latest", rust-target: "x86_64-pc-windows-msvc" }, ] steps: @@ -55,7 +54,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.platform.python-architecture }} + architecture: x64 - name: Install Rust toolchain uses: actions-rs/toolchain@v1 From 8eb657c4bf543a54104c1d5edab4df23547ad2de Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:39:54 +0100 Subject: [PATCH 09/77] update github actions --- .github/dependabot.yml | 5 +++++ .github/workflows/ci.yml | 34 ++++++++++------------------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c1ce3c3..8821fa7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,8 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddbfdf6..bf0e346 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: push: branches: - - master + - main pull_request: env: @@ -14,10 +14,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable - profile: minimal components: rustfmt - name: Check rust formatting (rustfmt) run: cargo fmt --all -- --check @@ -26,16 +24,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable - profile: minimal components: clippy - run: cargo clippy --all build: needs: [fmt] # don't wait for clippy as fails rarely and takes longer - name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} + name: python${{ matrix.python-version }} ${{ matrix.platform.os }} runs-on: ${{ matrix.platform.os }} strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. @@ -57,12 +53,9 @@ jobs: architecture: x64 - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: - toolchain: stable target: ${{ matrix.platform.rust-target }} - profile: minimal - default: true - name: Build without default features run: cargo test --no-default-features --verbose --target ${{ matrix.platform.rust-target }} @@ -83,21 +76,14 @@ jobs: target key: coverage-cargo-${{ hashFiles('**/Cargo.toml') }} continue-on-error: true - - name: install cargo-llvm-cov - run: | - wget https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz -qO- | tar -xzvf - - mv cargo-llvm-cov ~/.cargo/bin - env: - CARGO_LLVM_COV_VERSION: 0.1.9 - - uses: actions-rs/toolchain@v1 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: dtolnay/rust-toolchain@stable with: - toolchain: nightly - override: true - profile: minimal components: llvm-tools-preview - run: | cargo llvm-cov clean - cargo llvm-cov --lcov --output-path coverage.lcov + cargo llvm-cov --codecov --output-path codecov.json - uses: codecov/codecov-action@v2 with: - file: coverage.lcov + file: codecov.json From be2624b24465199d3b07d02314cce0d99c376e4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:11:34 +0000 Subject: [PATCH 10/77] build(deps): bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf0e346..04199af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt @@ -23,7 +23,7 @@ jobs: clippy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable with: components: clippy @@ -44,7 +44,7 @@ jobs: ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -67,7 +67,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/cache@v2 with: path: | From d728a9e7ecb3bbec777d622dd4855a68b7eace68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:11:41 +0000 Subject: [PATCH 11/77] build(deps): bump actions/setup-python from 2 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf0e346..ae1eee2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 From 13fd2e5787797e5372f50f05e673817ded8a1c9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:11:44 +0000 Subject: [PATCH 12/77] build(deps): bump codecov/codecov-action from 2 to 3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2 to 3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2...v3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf0e346..28558fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,6 @@ jobs: - run: | cargo llvm-cov clean cargo llvm-cov --codecov --output-path codecov.json - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 with: file: codecov.json From f88dade1cc3dd474bbab5bf4df712cf675911493 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:17:32 +0000 Subject: [PATCH 13/77] build(deps): bump actions/cache from 2 to 3 Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7937724..8c8ace1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: | ~/.cargo/registry From 8d010948fd4b60e24c5cbd149cb5295b06c3e730 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 30 Oct 2023 22:12:12 +0000 Subject: [PATCH 14/77] bump edition to 2021, add msrv 1.56, test coverage --- .github/workflows/ci.yml | 36 +++++++++++----------- CHANGELOG.md | 5 ++++ Cargo.toml | 3 +- src/de.rs | 11 ++++--- src/ser.rs | 65 +++++++++++++++++++++++++++++++++++----- 5 files changed, 88 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c8ace1..e4ff9a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,17 +31,22 @@ jobs: build: needs: [fmt] # don't wait for clippy as fails rarely and takes longer - name: python${{ matrix.python-version }} ${{ matrix.platform.os }} - runs-on: ${{ matrix.platform.os }} + name: python${{ matrix.python-version }} ${{ matrix.os }} rust-${{ matrix.rust}} + runs-on: ${{ matrix.os }} strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - platform: [ - { os: "macOS-latest", rust-target: "x86_64-apple-darwin" }, - { os: "ubuntu-latest", rust-target: "x86_64-unknown-linux-gnu" }, - { os: "windows-latest", rust-target: "x86_64-pc-windows-msvc" }, + os: [ + "macos-latest", + "ubuntu-latest", + "windows-latest", ] + rust: [stable] + include: + - python-version: "3.12" + os: "ubuntu-latest" + rust: "1.56" steps: - uses: actions/checkout@v4 @@ -53,12 +58,15 @@ jobs: architecture: x64 - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@master with: - target: ${{ matrix.platform.rust-target }} + toolchain: ${{ matrix.rust }} - - name: Build without default features - run: cargo test --no-default-features --verbose --target ${{ matrix.platform.rust-target }} + - uses: Swatinem/rust-cache@v2 + continue-on-error: true + + - name: Test + run: cargo test --verbose env: RUST_BACKTRACE: 1 @@ -68,13 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: coverage-cargo-${{ hashFiles('**/Cargo.toml') }} + - uses: Swatinem/rust-cache@v2 continue-on-error: true - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov diff --git a/CHANGELOG.md b/CHANGELOG.md index dab69ee..0318f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +- Bump edition to 2021 +- Bump MSRV to 1.56 + ## 0.20.0 - 2023-10-15 - Update to PyO3 0.20 diff --git a/Cargo.toml b/Cargo.toml index bbadfdd..229aef9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,8 @@ name = "pythonize" version = "0.20.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] -edition = "2018" +edition = "2021" +rust-version = "1.56" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" diff --git a/src/de.rs b/src/de.rs index 3cd181b..afef230 100644 --- a/src/de.rs +++ b/src/de.rs @@ -75,16 +75,12 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { self.deserialize_i64(visitor) } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::() { - self.deserialize_i64(visitor) } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.is_instance_of::() { self.deserialize_str(visitor) } else if obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::() { - self.deserialize_str(visitor) } else if obj.downcast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.downcast::().is_ok() { @@ -463,19 +459,22 @@ mod test { foo: String, bar: usize, baz: f32, + qux: bool, } let expected = Struct { foo: "Foo".to_string(), bar: 8usize, baz: 45.23, + qux: true, }; let expected_json = json!({ "foo": "Foo", "bar": 8, - "baz": 45.23 + "baz": 45.23, + "qux": true }); - let code = "{'foo': 'Foo', 'bar': 8, 'baz': 45.23}"; + let code = "{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"; test_de(code, &expected, &expected_json); } diff --git a/src/ser.rs b/src/ser.rs index 794498b..a112ea1 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -457,7 +457,7 @@ mod test { use maplit::hashmap; use pyo3::types::PyDict; use pyo3::{PyResult, Python}; - use serde::{Deserialize, Serialize}; + use serde::Serialize; fn test_ser(src: T, expected: &str) where @@ -486,7 +486,7 @@ mod test { #[test] fn test_empty_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct Empty; test_ser(Empty, "null"); @@ -494,7 +494,7 @@ mod test { #[test] fn test_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct Struct { foo: String, bar: usize, @@ -511,7 +511,7 @@ mod test { #[test] fn test_tuple_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct TupleStruct(String, usize); test_ser(TupleStruct("foo".to_string(), 5), r#"["foo",5]"#); @@ -534,7 +534,7 @@ mod test { #[test] fn test_enum_unit_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Empty, } @@ -544,7 +544,7 @@ mod test { #[test] fn test_enum_tuple_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Tuple(i32, String), } @@ -554,7 +554,7 @@ mod test { #[test] fn test_enum_newtype_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { NewType(String), } @@ -564,7 +564,7 @@ mod test { #[test] fn test_enum_struct_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Struct { foo: String, bar: usize }, } @@ -577,4 +577,53 @@ mod test { r#"{"Struct":{"foo":"foo","bar":5}}"#, ); } + + #[test] + fn test_integers() { + #[derive(Serialize)] + struct Integers { + a: i8, + b: i16, + c: i32, + d: i64, + e: u8, + f: u16, + g: u32, + h: u64, + } + + test_ser( + Integers { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + f: 6, + g: 7, + h: 8, + }, + r#"{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8}"#, + ) + } + + #[test] + fn test_bool() { + test_ser(true, "true"); + test_ser(false, "false"); + } + + #[test] + fn test_none() { + #[derive(Serialize)] + struct S; + + test_ser((), "null"); + test_ser(S, "null"); + } + + #[test] + fn test_bytes() { + test_ser(b"foo", "[102,111,111]"); + } } From 3abff6756b68dc911fcd7a5ab74eef0994c80ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:36:08 +0000 Subject: [PATCH 15/77] build(deps): bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4ff9a3..f7f74d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 From 94c7091b10ae3659ef2310ac60e5ac7fd26a74b0 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 14 Oct 2022 12:24:51 +0200 Subject: [PATCH 16/77] Expose the Pythonizer and Depythonizer --- src/de.rs | 2 ++ src/lib.rs | 4 ++-- src/ser.rs | 29 ++++++++++++++++++++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/de.rs b/src/de.rs index afef230..0867d24 100644 --- a/src/de.rs +++ b/src/de.rs @@ -13,11 +13,13 @@ where T::deserialize(&mut depythonizer) } +/// A structure that deserializes Python objects into Rust values pub struct Depythonizer<'de> { input: &'de PyAny, } impl<'de> Depythonizer<'de> { + /// Create a deserializer from a Python object pub fn from_object(input: &'de PyAny) -> Self { Depythonizer { input } } diff --git a/src/lib.rs b/src/lib.rs index 3e941b9..e39dba2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,8 +40,8 @@ mod de; mod error; mod ser; -pub use crate::de::depythonize; +pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, + pythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index a112ea1..96001e2 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -63,7 +63,7 @@ pub fn pythonize(py: Python, value: &T) -> Result where T: ?Sized + Serialize, { - pythonize_custom::(py, value) + value.serialize(Pythonizer::new(py)) } /// Attempt to convert the given data into a Python object. @@ -73,18 +73,37 @@ where T: ?Sized + Serialize, P: PythonizeTypes, { - value.serialize(Pythonizer::

{ - py, - _types: PhantomData, - }) + value.serialize(Pythonizer::custom::

(py)) } +/// A structure that serializes Rust values into Python objects #[derive(Clone, Copy)] pub struct Pythonizer<'py, P> { py: Python<'py>, _types: PhantomData

, } +impl<'py, P> From> for Pythonizer<'py, P> { + fn from(py: Python<'py>) -> Self { + Self { + py, + _types: PhantomData, + } + } +} + +impl<'py> Pythonizer<'py, PythonizeDefault> { + /// Creates a serializer to convert data into a Python object using the default mapping class + pub fn new(py: Python<'py>) -> Self { + Self::from(py) + } + + /// Creates a serializer to convert data into a Python object using a custom mapping class + pub fn custom

(py: Python<'py>) -> Pythonizer<'py, P> { + Pythonizer::from(py) + } +} + #[doc(hidden)] pub struct PythonCollectionSerializer<'py, P> { items: Vec, From f840ea7ed7bce7b5afcb3e7dc441cfdbc974b225 Mon Sep 17 00:00:00 2001 From: Samuel Collins Date: Mon, 29 Jan 2024 11:52:07 +0000 Subject: [PATCH 17/77] Add tests for integration with serde_path_to_err crate --- Cargo.toml | 1 + tests/test_with_serde_path_to_err.rs | 207 +++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 tests/test_with_serde_path_to_err.rs diff --git a/Cargo.toml b/Cargo.toml index 229aef9..9b0f57a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } pyo3 = { version = "0.20.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" +serde_path_to_error = "0.1.15" diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs new file mode 100644 index 0000000..da2340c --- /dev/null +++ b/tests/test_with_serde_path_to_err.rs @@ -0,0 +1,207 @@ +use std::collections::BTreeMap; + +use pyo3::{ + types::{PyDict, PyList}, + Py, PyAny, Python, +}; +use pythonize::PythonizeTypes; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +struct Root { + root_key: String, + root_map: BTreeMap>, +} + +impl PythonizeTypes for Root { + type Map = PyDict; + type List = PyList; +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +struct Nested { + nested_key: T, +} + +#[derive(Deserialize, Debug, PartialEq, Eq)] +struct CannotSerialize {} + +impl Serialize for CannotSerialize { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + Err(serde::ser::Error::custom( + "something went intentionally wrong", + )) + } +} + +#[test] +fn test_de_valid() { + Python::with_gil(|py| { + let pyroot = PyDict::new(py); + pyroot.set_item("root_key", "root_value").unwrap(); + + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); + nested_0.set_item("nested_key", "nested_value_0").unwrap(); + nested.set_item("nested_0", nested_0).unwrap(); + let nested_1 = PyDict::new(py); + nested_1.set_item("nested_key", "nested_value_1").unwrap(); + nested.set_item("nested_1", nested_1).unwrap(); + + pyroot.set_item("root_map", nested).unwrap(); + + let de = &mut pythonize::Depythonizer::from_object(pyroot); + let root: Root = serde_path_to_error::deserialize(de).unwrap(); + + assert_eq!( + root, + Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: String::from("nested_value_0") + } + ), + ( + String::from("nested_1"), + Nested { + nested_key: String::from("nested_value_1") + } + ) + ]) + } + ); + }) +} + +#[test] +fn test_de_invalid() { + Python::with_gil(|py| { + let pyroot = PyDict::new(py); + pyroot.set_item("root_key", "root_value").unwrap(); + + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); + nested_0.set_item("nested_key", "nested_value_0").unwrap(); + nested.set_item("nested_0", nested_0).unwrap(); + let nested_1 = PyDict::new(py); + nested_1.set_item("nested_key", 1).unwrap(); + nested.set_item("nested_1", nested_1).unwrap(); + + pyroot.set_item("root_map", nested).unwrap(); + + let de = &mut pythonize::Depythonizer::from_object(pyroot); + let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); + + assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); + assert_eq!(err.to_string(), "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be converted to 'PyString'"); + }) +} + +#[test] +fn test_ser_valid() { + Python::with_gil(|py| { + let root = Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: String::from("nested_value_0"), + }, + ), + ( + String::from("nested_1"), + Nested { + nested_key: String::from("nested_value_1"), + }, + ), + ]), + }; + + let ser = pythonize::Pythonizer::>::from(py); + let pyroot: Py = serde_path_to_error::serialize(&root, ser).unwrap(); + + let pyroot: &PyDict = pyroot.downcast(py).unwrap(); + assert_eq!(pyroot.len(), 2); + + let root_value: &str = pyroot + .get_item("root_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(root_value, "root_value"); + + let root_map: &PyDict = pyroot + .get_item("root_map") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(root_map.len(), 2); + + let nested_0: &PyDict = root_map + .get_item("nested_0") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_0.len(), 1); + let nested_key_0: &str = nested_0 + .get_item("nested_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_key_0, "nested_value_0"); + + let nested_1: &PyDict = root_map + .get_item("nested_1") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_1.len(), 1); + let nested_key_1: &str = nested_1 + .get_item("nested_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_key_1, "nested_value_1"); + }); +} + +#[test] +fn test_ser_invalid() { + Python::with_gil(|py| { + let root = Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: CannotSerialize {}, + }, + ), + ( + String::from("nested_1"), + Nested { + nested_key: CannotSerialize {}, + }, + ), + ]), + }; + + let ser = pythonize::Pythonizer::>::from(py); + let err = serde_path_to_error::serialize(&root, ser).unwrap_err(); + + assert_eq!(err.path().to_string(), "root_map.nested_0.nested_key"); + }); +} From 5bd87d9731cd4df7978ebe2e90166956554c66b7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 2 Feb 2024 07:32:44 +0000 Subject: [PATCH 18/77] rearrange `de` branches to sate clippy --- src/de.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/de.rs b/src/de.rs index 0867d24..2034b82 100644 --- a/src/de.rs +++ b/src/de.rs @@ -61,29 +61,31 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { { let obj = self.input; + // First check for cases which are cheap to check due to pointer + // comparison or bitflag checks if obj.is_none() { self.deserialize_unit(visitor) } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if obj.is_instance_of::() || obj.is_instance_of::() { - self.deserialize_bytes(visitor) - } else if obj.is_instance_of::() { - self.deserialize_map(visitor) - } else if obj.is_instance_of::() { - self.deserialize_f64(visitor) - } else if obj.is_instance_of::() { - self.deserialize_tuple(obj.len()?, visitor) } else if obj.is_instance_of::() { self.deserialize_i64(visitor) - } else if obj.is_instance_of::() { - self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::() { + } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) + } else if obj.is_instance_of::() { + self.deserialize_map(visitor) } else if obj.is_instance_of::() { self.deserialize_str(visitor) - } else if obj.is_instance_of::() { - self.deserialize_tuple(obj.len()?, visitor) - } else if obj.downcast::().is_ok() { + } + // Continue with cases which are slower to check because they go + // throuh `isinstance` machinery + else if obj.is_instance_of::() || obj.is_instance_of::() { + self.deserialize_bytes(visitor) + } else if obj.is_instance_of::() { + self.deserialize_f64(visitor) + } else if obj.is_instance_of::() + || obj.is_instance_of::() + || obj.downcast::().is_ok() + { self.deserialize_tuple(obj.len()?, visitor) } else if obj.downcast::().is_ok() { self.deserialize_map(visitor) From 7d703634d4800da024e2c4720296741d15848f28 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 18:56:47 +0000 Subject: [PATCH 19/77] add codecov token --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7f74d4..c90baa8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,3 +89,4 @@ jobs: - uses: codecov/codecov-action@v3 with: file: codecov.json + token: ${{ secrets.CODECOV_TOKEN }} From fb0d10d655c9ec1c06d5be9d322c97913720ec25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:57:44 +0000 Subject: [PATCH 20/77] build(deps): bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c90baa8..096dd32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: - run: | cargo llvm-cov clean cargo llvm-cov --codecov --output-path codecov.json - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 with: file: codecov.json token: ${{ secrets.CODECOV_TOKEN }} From 81aea334f6bf64245bec3bacb01092ecfab9de7a Mon Sep 17 00:00:00 2001 From: Gentle Date: Tue, 6 Feb 2024 21:56:58 +0100 Subject: [PATCH 21/77] Export PythonizeDefault for external crates this is needed to make Pythonizer work in other crates --- src/lib.rs | 3 ++- src/ser.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e39dba2..963d1bf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,5 +43,6 @@ mod ser; pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, + pythonize, pythonize_custom, PythonizeDefault, PythonizeDictType, PythonizeListType, + PythonizeTypes, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index 96001e2..3fddde6 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -51,7 +51,7 @@ impl PythonizeListType for PyList { } } -struct PythonizeDefault; +pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { type Map = PyDict; From ef42fe97ab0482d0032539ffe0b137f83602b7a3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 22:02:16 +0000 Subject: [PATCH 22/77] add test & changelog entry --- CHANGELOG.md | 1 + tests/test_custom_types.rs | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0318f3e..fe12e86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Bump edition to 2021 - Bump MSRV to 1.56 +- Export `PythonizeDefault` ## 0.20.0 - 2023-10-15 diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index e99bcbe..aacf978 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,8 +6,9 @@ use pyo3::{ types::{PyDict, PyList, PyMapping, PySequence}, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, + depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, }; +use serde::Serialize; use serde_json::{json, Value}; #[pyclass(sequence)] @@ -137,3 +138,22 @@ fn test_custom_dict() { assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); }) } + +#[test] +fn test_pythonizer_can_be_created() { + // https://github.com/davidhewitt/pythonize/pull/56 + Python::with_gil(|py| { + let sample = json!({ "hello": 1, "world": 2 }); + assert!(sample + .serialize(Pythonizer::new(py)) + .unwrap() + .as_ref(py) + .is_instance_of::()); + + assert!(sample + .serialize(Pythonizer::custom::(py)) + .unwrap() + .as_ref(py) + .is_instance_of::()); + }) +} From 2409260b8457b97c60bb3f1a3e7b19a615adfbc6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 22:47:35 +0000 Subject: [PATCH 23/77] update to PyO3 0.21 --- CHANGELOG.md | 1 + Cargo.toml | 4 +- src/de.rs | 140 +++++++++++++++------------ src/error.rs | 20 +++- src/lib.rs | 4 +- src/ser.rs | 42 ++++---- tests/test_custom_types.rs | 31 +++--- tests/test_with_serde_path_to_err.rs | 41 ++++---- 8 files changed, 165 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe12e86..79e361d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Bump edition to 2021 - Bump MSRV to 1.56 +- Update to PyO3 0.21 - Export `PythonizeDefault` ## 0.20.0 - 2023-10-15 diff --git a/Cargo.toml b/Cargo.toml index 9b0f57a..bf904ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.20.0", default-features = false } +pyo3 = { version = "0.21.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.20.0", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.21.0", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" serde_path_to_error = "0.1.15" diff --git a/src/de.rs b/src/de.rs index 2034b82..573e17c 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,42 +1,64 @@ -use pyo3::types::*; +use pyo3::{types::*, Bound, PyNativeType}; use serde::de::{self, IntoDeserializer}; use serde::Deserialize; use crate::error::{PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` +#[deprecated( + since = "0.21.0", + note = "will be replaced by `depythonize_bound` in a future release" +)] pub fn depythonize<'de, T>(obj: &'de PyAny) -> Result where T: Deserialize<'de>, { - let mut depythonizer = Depythonizer::from_object(obj); + let mut depythonizer = Depythonizer::from_object_bound(obj.as_borrowed().to_owned()); + T::deserialize(&mut depythonizer) +} + +/// Attempt to convert a Python object to an instance of `T` +pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result +where + T: for<'a> Deserialize<'a>, +{ + let mut depythonizer = Depythonizer::from_object_bound(obj); T::deserialize(&mut depythonizer) } /// A structure that deserializes Python objects into Rust values -pub struct Depythonizer<'de> { - input: &'de PyAny, +pub struct Depythonizer<'py> { + input: Bound<'py, PyAny>, } -impl<'de> Depythonizer<'de> { +impl<'py> Depythonizer<'py> { + /// Create a deserializer from a Python object + #[deprecated( + since = "0.21.0", + note = "will be replaced by `Depythonizer::from_object_bound` in a future version" + )] + pub fn from_object(input: &'py PyAny) -> Self { + Self::from_object_bound(input.as_borrowed().to_owned()) + } + /// Create a deserializer from a Python object - pub fn from_object(input: &'de PyAny) -> Self { + pub fn from_object_bound(input: Bound<'py, PyAny>) -> Self { Depythonizer { input } } - fn sequence_access(&self, expected_len: Option) -> Result> { - let seq: &PySequence = self.input.downcast()?; + fn sequence_access(&self, expected_len: Option) -> Result> { + let seq = self.input.downcast::()?; let len = self.input.len()?; match expected_len { Some(expected) if expected != len => { Err(PythonizeError::incorrect_sequence_length(expected, len)) } - _ => Ok(PySequenceAccess::new(seq, len)), + _ => Ok(PySequenceAccess::new(seq.clone(), len)), } } - fn dict_access(&self) -> Result> { + fn dict_access(&self) -> Result> { PyMappingAccess::new(self.input.downcast()?) } } @@ -52,14 +74,14 @@ macro_rules! deserialize_type { }; } -impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { +impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { type Error = PythonizeError; fn deserialize_any(self, visitor: V) -> Result where V: de::Visitor<'de>, { - let obj = self.input; + let obj = &self.input; // First check for cases which are cheap to check due to pointer // comparison or bitflag checks @@ -90,8 +112,9 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { } else if obj.downcast::().is_ok() { self.deserialize_map(visitor) } else { - Err(PythonizeError::unsupported_type( - obj.get_type().name().unwrap_or(""), + Err(obj.get_type().qualname().map_or_else( + |_| PythonizeError::unsupported_type("unknown"), + PythonizeError::unsupported_type, )) } } @@ -100,7 +123,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - visitor.visit_bool(self.input.is_true()?) + visitor.visit_bool(self.input.is_truthy()?) } fn deserialize_char(self, visitor: V) -> Result @@ -129,7 +152,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s: &PyString = self.input.downcast()?; + let s = self.input.downcast::()?; visitor.visit_str(s.to_str()?) } @@ -144,8 +167,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let obj = self.input; - let b: &PyBytes = obj.downcast()?; + let b = self.input.downcast::()?; visitor.visit_bytes(b.as_bytes()) } @@ -246,23 +268,21 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let item = self.input; - if item.is_instance_of::() { + let item = &self.input; + if let Ok(d) = item.downcast::() { // Get the enum variant from the dict key - let d: &PyDict = item.downcast().unwrap(); if d.len() != 1 { return Err(PythonizeError::invalid_length_enum()); } - let variant: &PyString = d + let variant = d .keys() .get_item(0)? - .downcast() + .downcast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; - let value = d.get_item(variant)?.unwrap(); - let mut de = Depythonizer::from_object(value); + let value = d.get_item(&variant)?.unwrap(); + let mut de = Depythonizer::from_object_bound(value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) - } else if item.is_instance_of::() { - let s: &PyString = self.input.downcast()?; + } else if let Ok(s) = item.downcast::() { visitor.visit_enum(s.to_str()?.into_deserializer()) } else { Err(PythonizeError::invalid_enum_type()) @@ -273,9 +293,9 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s: &PyString = self + let s = self .input - .downcast() + .downcast::() .map_err(|_| PythonizeError::dict_key_not_string())?; visitor.visit_str(s.to_str()?) } @@ -288,19 +308,19 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { } } -struct PySequenceAccess<'a> { - seq: &'a PySequence, +struct PySequenceAccess<'py> { + seq: Bound<'py, PySequence>, index: usize, len: usize, } -impl<'a> PySequenceAccess<'a> { - fn new(seq: &'a PySequence, len: usize) -> Self { +impl<'py> PySequenceAccess<'py> { + fn new(seq: Bound<'py, PySequence>, len: usize) -> Self { Self { seq, index: 0, len } } } -impl<'de> de::SeqAccess<'de> for PySequenceAccess<'de> { +impl<'de, 'py> de::SeqAccess<'de> for PySequenceAccess<'py> { type Error = PythonizeError; fn next_element_seed(&mut self, seed: T) -> Result> @@ -308,7 +328,7 @@ impl<'de> de::SeqAccess<'de> for PySequenceAccess<'de> { T: de::DeserializeSeed<'de>, { if self.index < self.len { - let mut item_de = Depythonizer::from_object(self.seq.get_item(self.index)?); + let mut item_de = Depythonizer::from_object_bound(self.seq.get_item(self.index)?); self.index += 1; seed.deserialize(&mut item_de).map(Some) } else { @@ -317,16 +337,16 @@ impl<'de> de::SeqAccess<'de> for PySequenceAccess<'de> { } } -struct PyMappingAccess<'de> { - keys: &'de PySequence, - values: &'de PySequence, +struct PyMappingAccess<'py> { + keys: Bound<'py, PySequence>, + values: Bound<'py, PySequence>, key_idx: usize, val_idx: usize, len: usize, } -impl<'de> PyMappingAccess<'de> { - fn new(map: &'de PyMapping) -> Result { +impl<'py> PyMappingAccess<'py> { + fn new(map: &Bound<'py, PyMapping>) -> Result { let keys = map.keys()?; let values = map.values()?; let len = map.len()?; @@ -340,7 +360,7 @@ impl<'de> PyMappingAccess<'de> { } } -impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { +impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { type Error = PythonizeError; fn next_key_seed(&mut self, seed: K) -> Result> @@ -348,7 +368,7 @@ impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { K: de::DeserializeSeed<'de>, { if self.key_idx < self.len { - let mut item_de = Depythonizer::from_object(self.keys.get_item(self.key_idx)?); + let mut item_de = Depythonizer::from_object_bound(self.keys.get_item(self.key_idx)?); self.key_idx += 1; seed.deserialize(&mut item_de).map(Some) } else { @@ -360,24 +380,24 @@ impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { where V: de::DeserializeSeed<'de>, { - let mut item_de = Depythonizer::from_object(self.values.get_item(self.val_idx)?); + let mut item_de = Depythonizer::from_object_bound(self.values.get_item(self.val_idx)?); self.val_idx += 1; seed.deserialize(&mut item_de) } } -struct PyEnumAccess<'a, 'de> { - de: &'a mut Depythonizer<'de>, - variant: &'de PyString, +struct PyEnumAccess<'a, 'py> { + de: &'a mut Depythonizer<'py>, + variant: Bound<'py, PyString>, } -impl<'a, 'de> PyEnumAccess<'a, 'de> { - fn new(de: &'a mut Depythonizer<'de>, variant: &'de PyString) -> Self { +impl<'a, 'py> PyEnumAccess<'a, 'py> { + fn new(de: &'a mut Depythonizer<'py>, variant: Bound<'py, PyString>) -> Self { Self { de, variant } } } -impl<'a, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'de> { +impl<'a, 'py, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'py> { type Error = PythonizeError; type Variant = Self; @@ -392,7 +412,7 @@ impl<'a, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'de> { } } -impl<'a, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'de> { +impl<'a, 'py, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'py> { type Error = PythonizeError; fn unit_variant(self) -> Result<()> { @@ -434,13 +454,13 @@ mod test { T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) + let locals = PyDict::new_bound(py); + py.run_bound(&format!("obj = {}", code), None, Some(&locals)) .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); - let actual: T = depythonize(obj).unwrap(); + let actual: T = depythonize_bound(obj.clone()).unwrap(); assert_eq!(&actual, expected); - let actual_json: JsonValue = depythonize(obj).unwrap(); + let actual_json: JsonValue = depythonize_bound(obj).unwrap(); assert_eq!(&actual_json, expected_json); }); } @@ -493,12 +513,12 @@ mod test { let code = "{'foo': 'Foo'}"; Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) + let locals = PyDict::new_bound(py); + py.run_bound(&format!("obj = {}", code), None, Some(&locals)) .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( - *depythonize::(obj).unwrap_err().inner, + *depythonize_bound::(obj).unwrap_err().inner, ErrorImpl::Message(msg) if msg == "missing field `bar`" )); }) @@ -523,12 +543,12 @@ mod test { let code = "('cat', -10.05, 'foo')"; Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) + let locals = PyDict::new_bound(py); + py.run_bound(&format!("obj = {}", code), None, Some(&locals)) .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( - *depythonize::(obj).unwrap_err().inner, + *depythonize_bound::(obj).unwrap_err().inner, ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 )); }) diff --git a/src/error.rs b/src/error.rs index a86169e..d3ddd71 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,4 +1,4 @@ -use pyo3::exceptions::*; +use pyo3::{exceptions::*, DowncastError, DowncastIntoError}; use pyo3::{PyDowncastError, PyErr}; use serde::{de, ser}; use std::error; @@ -154,6 +154,24 @@ impl<'a> From> for PythonizeError { } } +/// Handle errors that occur when attempting to use `PyAny::cast_as` +impl<'a, 'py> From> for PythonizeError { + fn from(other: DowncastError<'a, 'py>) -> Self { + Self { + inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), + } + } +} + +/// Handle errors that occur when attempting to use `PyAny::cast_as` +impl<'py> From> for PythonizeError { + fn from(other: DowncastIntoError<'py>) -> Self { + Self { + inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), + } + } +} + /// Convert a `PythonizeError` to a Python exception impl From for PyErr { fn from(other: PythonizeError) -> Self { diff --git a/src/lib.rs b/src/lib.rs index 963d1bf..ea04c22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,9 @@ mod de; mod error; mod ser; -pub use crate::de::{depythonize, Depythonizer}; +#[allow(deprecated)] +pub use crate::de::depythonize; +pub use crate::de::{depythonize_bound, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ pythonize, pythonize_custom, PythonizeDefault, PythonizeDictType, PythonizeListType, diff --git a/src/ser.rs b/src/ser.rs index 3fddde6..d04ac8c 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; -use pyo3::types::{PyDict, PyList, PyMapping, PySequence, PyTuple}; -use pyo3::{IntoPy, PyObject, PyResult, Python, ToPyObject}; +use pyo3::types::{PyAnyMethods, PyDict, PyList, PyMapping, PySequence, PyTuple}; +use pyo3::{Bound, IntoPy, PyObject, PyResult, Python, ToPyObject}; use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; @@ -9,7 +9,7 @@ use crate::error::{PythonizeError, Result}; /// Trait for types which can represent a Python mapping pub trait PythonizeDictType { /// Constructor - fn create_mapping(py: Python) -> PyResult<&PyMapping>; + fn create_mapping(py: Python) -> PyResult>; } /// Trait for types which can represent a Python sequence @@ -18,7 +18,7 @@ pub trait PythonizeListType: Sized { fn create_sequence( py: Python, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where T: ToPyObject, U: ExactSizeIterator; @@ -33,8 +33,8 @@ pub trait PythonizeTypes { } impl PythonizeDictType for PyDict { - fn create_mapping(py: Python) -> PyResult<&PyMapping> { - Ok(PyDict::new(py).as_mapping()) + fn create_mapping(py: Python) -> PyResult> { + Ok(PyDict::new_bound(py).into_any().downcast_into().unwrap()) } } @@ -42,12 +42,15 @@ impl PythonizeListType for PyList { fn create_sequence( py: Python, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where T: ToPyObject, U: ExactSizeIterator, { - Ok(PyList::new(py, elements).as_sequence()) + Ok(PyList::new_bound(py, elements) + .into_any() + .downcast_into() + .unwrap()) } } @@ -126,14 +129,14 @@ pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { #[doc(hidden)] pub struct PythonDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - dict: &'py PyMapping, + dict: Bound<'py, PyMapping>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - map: &'py PyMapping, + map: Bound<'py, PyMapping>, key: Option, _types: PhantomData

, } @@ -250,7 +253,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { where T: ?Sized + Serialize, { - let d = PyDict::new(self.py); + let d = PyDict::new_bound(self.py); d.set_item(variant, value.serialize(self)?)?; Ok(d.into()) } @@ -363,7 +366,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer< } fn end(self) -> Result { - Ok(PyTuple::new(self.py, self.items).into()) + Ok(PyTuple::new_bound(self.py, self.items).into()) } } @@ -395,7 +398,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe } fn end(self) -> Result { - let d = PyDict::new(self.inner.py); + let d = PyDict::new_bound(self.inner.py); d.set_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; Ok(d.into()) } @@ -464,7 +467,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant } fn end(self) -> Result { - let d = PyDict::new(self.inner.py); + let d = PyDict::new_bound(self.inner.py); d.set_item(self.variant, self.inner.dict)?; Ok(d.into()) } @@ -474,8 +477,8 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant mod test { use super::pythonize; use maplit::hashmap; + use pyo3::prelude::*; use pyo3::types::PyDict; - use pyo3::{PyResult, Python}; use serde::Serialize; fn test_ser(src: T, expected: &str) @@ -485,15 +488,16 @@ mod test { Python::with_gil(|py| -> PyResult<()> { let obj = pythonize(py, &src)?; - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("obj", obj)?; - py.run( + py.run_bound( "import json; result = json.dumps(obj, separators=(',', ':'))", None, - Some(locals), + Some(&locals), )?; - let result = locals.get_item("result")?.unwrap().extract::<&str>()?; + let result = locals.get_item("result")?.unwrap(); + let result = result.extract::<&str>()?; assert_eq!(result, expected); assert_eq!(serde_json::to_string(&src).unwrap(), expected); diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index aacf978..bcadabd 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,7 +6,8 @@ use pyo3::{ types::{PyDict, PyList, PyMapping, PySequence}, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, + depythonize_bound, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, + Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -34,12 +35,12 @@ impl PythonizeListType for CustomList { fn create_sequence( py: Python, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where T: ToPyObject, U: ExactSizeIterator, { - let sequence = Py::new( + let sequence = Bound::new( py, CustomList { items: elements @@ -48,9 +49,9 @@ impl PythonizeListType for CustomList { .collect(), }, )? - .into_ref(py); + .into_any(); - Ok(unsafe { PySequence::try_from_unchecked(sequence.as_ref()) }) + Ok(unsafe { sequence.downcast_into_unchecked() }) } } @@ -66,10 +67,10 @@ fn test_custom_list() { PySequence::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!([1, 2, 3])) .unwrap() - .into_ref(py); + .into_bound(py); assert!(serialized.is_instance_of::()); - let deserialized: Value = depythonize(serialized).unwrap(); + let deserialized: Value = depythonize_bound(serialized).unwrap(); assert_eq!(deserialized, json!([1, 2, 3])); }) } @@ -106,15 +107,15 @@ impl CustomDict { } impl PythonizeDictType for CustomDict { - fn create_mapping(py: Python) -> PyResult<&PyMapping> { - let mapping = Py::new( + fn create_mapping(py: Python) -> PyResult> { + let mapping = Bound::new( py, CustomDict { items: HashMap::new(), }, )? - .into_ref(py); - Ok(unsafe { PyMapping::try_from_unchecked(mapping.as_ref()) }) + .into_any(); + Ok(unsafe { mapping.downcast_into_unchecked() }) } } @@ -131,10 +132,10 @@ fn test_custom_dict() { let serialized = pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) .unwrap() - .into_ref(py); + .into_bound(py); assert!(serialized.is_instance_of::()); - let deserialized: Value = depythonize(serialized).unwrap(); + let deserialized: Value = depythonize_bound(serialized).unwrap(); assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); }) } @@ -147,13 +148,13 @@ fn test_pythonizer_can_be_created() { assert!(sample .serialize(Pythonizer::new(py)) .unwrap() - .as_ref(py) + .bind(py) .is_instance_of::()); assert!(sample .serialize(Pythonizer::custom::(py)) .unwrap() - .as_ref(py) + .bind(py) .is_instance_of::()); }) } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index da2340c..6b2c4aa 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use pyo3::{ + prelude::*, types::{PyDict, PyList}, Py, PyAny, Python, }; @@ -40,20 +41,20 @@ impl Serialize for CannotSerialize { #[test] fn test_de_valid() { Python::with_gil(|py| { - let pyroot = PyDict::new(py); + let pyroot = PyDict::new_bound(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new(py); - let nested_0 = PyDict::new(py); + let nested = PyDict::new_bound(py); + let nested_0 = PyDict::new_bound(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new(py); + let nested_1 = PyDict::new_bound(py); nested_1.set_item("nested_key", "nested_value_1").unwrap(); nested.set_item("nested_1", nested_1).unwrap(); pyroot.set_item("root_map", nested).unwrap(); - let de = &mut pythonize::Depythonizer::from_object(pyroot); + let de = &mut pythonize::Depythonizer::from_object_bound(pyroot.into_any()); let root: Root = serde_path_to_error::deserialize(de).unwrap(); assert_eq!( @@ -82,20 +83,20 @@ fn test_de_valid() { #[test] fn test_de_invalid() { Python::with_gil(|py| { - let pyroot = PyDict::new(py); + let pyroot = PyDict::new_bound(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new(py); - let nested_0 = PyDict::new(py); + let nested = PyDict::new_bound(py); + let nested_0 = PyDict::new_bound(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new(py); + let nested_1 = PyDict::new_bound(py); nested_1.set_item("nested_key", 1).unwrap(); nested.set_item("nested_1", nested_1).unwrap(); pyroot.set_item("root_map", nested).unwrap(); - let de = &mut pythonize::Depythonizer::from_object(pyroot); + let de = &mut pythonize::Depythonizer::from_object_bound(pyroot.into_any()); let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); @@ -127,10 +128,10 @@ fn test_ser_valid() { let ser = pythonize::Pythonizer::>::from(py); let pyroot: Py = serde_path_to_error::serialize(&root, ser).unwrap(); - let pyroot: &PyDict = pyroot.downcast(py).unwrap(); + let pyroot = pyroot.bind(py).downcast::().unwrap(); assert_eq!(pyroot.len(), 2); - let root_value: &str = pyroot + let root_value: String = pyroot .get_item("root_key") .unwrap() .unwrap() @@ -138,22 +139,22 @@ fn test_ser_valid() { .unwrap(); assert_eq!(root_value, "root_value"); - let root_map: &PyDict = pyroot + let root_map = pyroot .get_item("root_map") .unwrap() .unwrap() - .extract() + .downcast_into::() .unwrap(); assert_eq!(root_map.len(), 2); - let nested_0: &PyDict = root_map + let nested_0 = root_map .get_item("nested_0") .unwrap() .unwrap() - .extract() + .downcast_into::() .unwrap(); assert_eq!(nested_0.len(), 1); - let nested_key_0: &str = nested_0 + let nested_key_0: String = nested_0 .get_item("nested_key") .unwrap() .unwrap() @@ -161,14 +162,14 @@ fn test_ser_valid() { .unwrap(); assert_eq!(nested_key_0, "nested_value_0"); - let nested_1: &PyDict = root_map + let nested_1 = root_map .get_item("nested_1") .unwrap() .unwrap() - .extract() + .downcast_into::() .unwrap(); assert_eq!(nested_1.len(), 1); - let nested_key_1: &str = nested_1 + let nested_key_1: String = nested_1 .get_item("nested_key") .unwrap() .unwrap() From 16f17a9b42bda82c5006cbf6ab710f76474581bb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 1 Apr 2024 16:17:36 +0100 Subject: [PATCH 24/77] release: 0.21.0 --- CHANGELOG.md | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e361d..1d27ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.21.0 - 2024-04-01 - Bump edition to 2021 - Bump MSRV to 1.56 diff --git a/Cargo.toml b/Cargo.toml index bf904ed..61df375 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.20.0" +version = "0.21.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.56" From 1b1048993608f1102a7fcf1cd3bf2fc669b717ee Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 2 Apr 2024 20:27:42 +0100 Subject: [PATCH 25/77] fix: `to_str` not available on all abi3 builds --- .github/workflows/ci.yml | 3 +++ CHANGELOG.md | 4 ++++ Cargo.toml | 2 +- src/de.rs | 12 ++++++------ src/ser.rs | 3 ++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 096dd32..9ec1f92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,9 @@ jobs: - name: Test run: cargo test --verbose + - name: Test (abi3) + run: cargo test --verbose --features pyo3/abi3-py37 + env: RUST_BACKTRACE: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d27ca5..5ebb355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.21.1 - 2024-04-02 + +- Fix compile error when using PyO3 `abi3` feature targeting a minimum version below 3.10 + ## 0.21.0 - 2024-04-01 - Bump edition to 2021 diff --git a/Cargo.toml b/Cargo.toml index 61df375..938c95a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ pyo3 = { version = "0.21.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.21.0", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.21.1", default-features = false, features = ["auto-initialize", "macros"] } serde_json = "1.0" maplit = "1.0.2" serde_path_to_error = "0.1.15" diff --git a/src/de.rs b/src/de.rs index 573e17c..c7174fd 100644 --- a/src/de.rs +++ b/src/de.rs @@ -130,7 +130,7 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { where V: de::Visitor<'de>, { - let s = self.input.downcast::()?.to_str()?; + let s = self.input.downcast::()?.to_cow()?; if s.len() != 1 { return Err(PythonizeError::invalid_length_char()); } @@ -153,7 +153,7 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { V: de::Visitor<'de>, { let s = self.input.downcast::()?; - visitor.visit_str(s.to_str()?) + visitor.visit_str(&s.to_cow()?) } fn deserialize_string(self, visitor: V) -> Result @@ -283,7 +283,7 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { let mut de = Depythonizer::from_object_bound(value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) } else if let Ok(s) = item.downcast::() { - visitor.visit_enum(s.to_str()?.into_deserializer()) + visitor.visit_enum(s.to_cow()?.into_deserializer()) } else { Err(PythonizeError::invalid_enum_type()) } @@ -297,7 +297,7 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { .input .downcast::() .map_err(|_| PythonizeError::dict_key_not_string())?; - visitor.visit_str(s.to_str()?) + visitor.visit_str(&s.to_cow()?) } fn deserialize_ignored_any(self, visitor: V) -> Result @@ -405,8 +405,8 @@ impl<'a, 'py, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'py> { where V: de::DeserializeSeed<'de>, { - let de: de::value::StrDeserializer<'_, PythonizeError> = - self.variant.to_str()?.into_deserializer(); + let cow = self.variant.to_cow()?; + let de: de::value::StrDeserializer<'_, PythonizeError> = cow.as_ref().into_deserializer(); let val = seed.deserialize(de)?; Ok((val, self)) } diff --git a/src/ser.rs b/src/ser.rs index d04ac8c..ba7fb30 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -478,6 +478,7 @@ mod test { use super::pythonize; use maplit::hashmap; use pyo3::prelude::*; + use pyo3::pybacked::PyBackedStr; use pyo3::types::PyDict; use serde::Serialize; @@ -497,7 +498,7 @@ mod test { Some(&locals), )?; let result = locals.get_item("result")?.unwrap(); - let result = result.extract::<&str>()?; + let result = result.extract::()?; assert_eq!(result, expected); assert_eq!(serde_json::to_string(&src).unwrap(), expected); From c3a45879ce0455099191c4de26bd62d2743cd7d2 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 2 Apr 2024 20:47:26 +0100 Subject: [PATCH 26/77] release: 0.21.1 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 938c95a..79b8a91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.21.0" +version = "0.21.1" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.56" From f56a9790ff5c7f09e6b6791087fdbdf3cef7592d Mon Sep 17 00:00:00 2001 From: Samuel Lijin Date: Fri, 5 Apr 2024 15:50:15 -0700 Subject: [PATCH 27/77] docs: update readme example of depythonize --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 78170fb..624e22d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,16 @@ that which is produced directly by `pythonize`. This crate converts Rust types which implement the [Serde] serialization traits into Python objects using the [PyO3] library. -Pythonize has two public APIs: `pythonize` and `depythonize`. +Pythonize has two public APIs: `pythonize` and `depythonize_bound`. + + +

+ +⚠️ Warning: API update in progress 🛠️ + +PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`, and pythonize is doing the same. + +
[Serde]: https://github.com/serde-rs/serde [PyO3]: https://github.com/PyO3/pyo3 @@ -20,7 +29,7 @@ Pythonize has two public APIs: `pythonize` and `depythonize`. ```rust use serde::{Serialize, Deserialize}; use pyo3::Python; -use pythonize::{depythonize, pythonize}; +use pythonize::{depythonize_bound, pythonize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct Sample { @@ -37,12 +46,12 @@ let sample = Sample { }; // Rust -> Python -let obj = pythonize(py, &sample).unwrap(); +let obj = pythonize(py, &sample).unwrap(); assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); // Python -> Rust -let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); +let new_sample: Sample = depythonize_bound(obj.into_bound(py)).unwrap(); assert_eq!(new_sample, sample); ``` From b2138b3c399a664208725b0c0327726b7cf0c6df Mon Sep 17 00:00:00 2001 From: Antoine PLASKOWSKI Date: Sun, 16 Jun 2024 00:18:09 +0200 Subject: [PATCH 28/77] Change Bound to &Bound --- CHANGELOG.md | 9 +++ src/de.rs | 99 ++++++++++++++++------------ src/lib.rs | 6 +- tests/test_custom_types.rs | 7 +- tests/test_with_serde_path_to_err.rs | 4 +- 5 files changed, 73 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ebb355..d9356e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## Unreleased + +- `depythonize()` now take a `&Bound` and is no longer depreciate +- `depythonize_object()` replace the old `depythonize()` and is depreciated +- `depythonize_bound()` is depreciated +- `Depythonizer` now need a `&Bound` and so have extra lifetime `'bound` +- `Depythonizer::from_object()` now take a `&Bound` and is no longer depreciate +- `Depythonizer::from_object_bound()` can't be implemented so have been removed + ## 0.21.1 - 2024-04-02 - Fix compile error when using PyO3 `abi3` feature targeting a minimum version below 3.10 diff --git a/src/de.rs b/src/de.rs index c7174fd..62bb7cb 100644 --- a/src/de.rs +++ b/src/de.rs @@ -5,48 +5,58 @@ use serde::Deserialize; use crate::error::{PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` -#[deprecated( - since = "0.21.0", - note = "will be replaced by `depythonize_bound` in a future release" -)] -pub fn depythonize<'de, T>(obj: &'de PyAny) -> Result +pub fn depythonize<'py, 'obj, T>(obj: &'obj Bound<'py, PyAny>) -> Result where - T: Deserialize<'de>, + T: Deserialize<'obj>, { - let mut depythonizer = Depythonizer::from_object_bound(obj.as_borrowed().to_owned()); + let mut depythonizer = Depythonizer::from_object(obj); T::deserialize(&mut depythonizer) } /// Attempt to convert a Python object to an instance of `T` +#[deprecated( + since = "0.21.1", + note = "will be replaced by `depythonize` in a future release" +)] pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result where T: for<'a> Deserialize<'a>, { - let mut depythonizer = Depythonizer::from_object_bound(obj); + let mut depythonizer = Depythonizer::from_object(&obj); T::deserialize(&mut depythonizer) } -/// A structure that deserializes Python objects into Rust values -pub struct Depythonizer<'py> { - input: Bound<'py, PyAny>, +/// Attempt to convert a Python object to an instance of `T` +#[deprecated( + since = "0.21.1", + note = "will be replaced by `depythonize` in a future release" +)] +pub fn depythonize_object<'de, T>(obj: &'de PyAny) -> Result +where + T: Deserialize<'de>, +{ + let obj = obj.as_borrowed().to_owned(); + let mut depythonizer = Depythonizer::from_object(&obj); + T::deserialize(&mut depythonizer) } -impl<'py> Depythonizer<'py> { - /// Create a deserializer from a Python object - #[deprecated( - since = "0.21.0", - note = "will be replaced by `Depythonizer::from_object_bound` in a future version" - )] - pub fn from_object(input: &'py PyAny) -> Self { - Self::from_object_bound(input.as_borrowed().to_owned()) - } +/// A structure that deserializes Python objects into Rust values +pub struct Depythonizer<'py, 'bound> { + input: &'bound Bound<'py, PyAny>, +} +impl<'py, 'bound> Depythonizer<'py, 'bound> { /// Create a deserializer from a Python object - pub fn from_object_bound(input: Bound<'py, PyAny>) -> Self { + pub fn from_object<'input, 'gil>( + input: &'input Bound<'gil, PyAny>, + ) -> Depythonizer<'gil, 'input> { Depythonizer { input } } - fn sequence_access(&self, expected_len: Option) -> Result> { + fn sequence_access( + &self, + expected_len: Option, + ) -> Result> { let seq = self.input.downcast::()?; let len = self.input.len()?; @@ -54,7 +64,7 @@ impl<'py> Depythonizer<'py> { Some(expected) if expected != len => { Err(PythonizeError::incorrect_sequence_length(expected, len)) } - _ => Ok(PySequenceAccess::new(seq.clone(), len)), + _ => Ok(PySequenceAccess::new(seq, len)), } } @@ -74,7 +84,7 @@ macro_rules! deserialize_type { }; } -impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { +impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, 'bound> { type Error = PythonizeError; fn deserialize_any(self, visitor: V) -> Result @@ -280,7 +290,7 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { .downcast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; let value = d.get_item(&variant)?.unwrap(); - let mut de = Depythonizer::from_object_bound(value); + let mut de = Depythonizer::from_object(&value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) } else if let Ok(s) = item.downcast::() { visitor.visit_enum(s.to_cow()?.into_deserializer()) @@ -308,19 +318,19 @@ impl<'a, 'py, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'py> { } } -struct PySequenceAccess<'py> { - seq: Bound<'py, PySequence>, +struct PySequenceAccess<'py, 'bound> { + seq: &'bound Bound<'py, PySequence>, index: usize, len: usize, } -impl<'py> PySequenceAccess<'py> { - fn new(seq: Bound<'py, PySequence>, len: usize) -> Self { +impl<'py, 'bound> PySequenceAccess<'py, 'bound> { + fn new(seq: &'bound Bound<'py, PySequence>, len: usize) -> Self { Self { seq, index: 0, len } } } -impl<'de, 'py> de::SeqAccess<'de> for PySequenceAccess<'py> { +impl<'de, 'py, 'bound> de::SeqAccess<'de> for PySequenceAccess<'py, 'bound> { type Error = PythonizeError; fn next_element_seed(&mut self, seed: T) -> Result> @@ -328,7 +338,8 @@ impl<'de, 'py> de::SeqAccess<'de> for PySequenceAccess<'py> { T: de::DeserializeSeed<'de>, { if self.index < self.len { - let mut item_de = Depythonizer::from_object_bound(self.seq.get_item(self.index)?); + let item = self.seq.get_item(self.index)?; + let mut item_de = Depythonizer::from_object(&item); self.index += 1; seed.deserialize(&mut item_de).map(Some) } else { @@ -368,7 +379,8 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { K: de::DeserializeSeed<'de>, { if self.key_idx < self.len { - let mut item_de = Depythonizer::from_object_bound(self.keys.get_item(self.key_idx)?); + let item = self.keys.get_item(self.key_idx)?; + let mut item_de = Depythonizer::from_object(&item); self.key_idx += 1; seed.deserialize(&mut item_de).map(Some) } else { @@ -380,24 +392,25 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { where V: de::DeserializeSeed<'de>, { - let mut item_de = Depythonizer::from_object_bound(self.values.get_item(self.val_idx)?); + let item = self.values.get_item(self.val_idx)?; + let mut item_de = Depythonizer::from_object(&item); self.val_idx += 1; seed.deserialize(&mut item_de) } } -struct PyEnumAccess<'a, 'py> { - de: &'a mut Depythonizer<'py>, +struct PyEnumAccess<'a, 'py, 'bound> { + de: &'a mut Depythonizer<'py, 'bound>, variant: Bound<'py, PyString>, } -impl<'a, 'py> PyEnumAccess<'a, 'py> { - fn new(de: &'a mut Depythonizer<'py>, variant: Bound<'py, PyString>) -> Self { +impl<'a, 'py, 'bound> PyEnumAccess<'a, 'py, 'bound> { + fn new(de: &'a mut Depythonizer<'py, 'bound>, variant: Bound<'py, PyString>) -> Self { Self { de, variant } } } -impl<'a, 'py, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'py> { +impl<'a, 'py, 'de, 'bound> de::EnumAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { type Error = PythonizeError; type Variant = Self; @@ -412,7 +425,7 @@ impl<'a, 'py, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'py> { } } -impl<'a, 'py, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'py> { +impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { type Error = PythonizeError; fn unit_variant(self) -> Result<()> { @@ -458,9 +471,9 @@ mod test { py.run_bound(&format!("obj = {}", code), None, Some(&locals)) .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); - let actual: T = depythonize_bound(obj.clone()).unwrap(); + let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); - let actual_json: JsonValue = depythonize_bound(obj).unwrap(); + let actual_json: JsonValue = depythonize(&obj).unwrap(); assert_eq!(&actual_json, expected_json); }); } @@ -518,7 +531,7 @@ mod test { .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( - *depythonize_bound::(obj).unwrap_err().inner, + *depythonize::(&obj).unwrap_err().inner, ErrorImpl::Message(msg) if msg == "missing field `bar`" )); }) @@ -548,7 +561,7 @@ mod test { .unwrap(); let obj = locals.get_item("obj").unwrap().unwrap(); assert!(matches!( - *depythonize_bound::(obj).unwrap_err().inner, + *depythonize::(&obj).unwrap_err().inner, ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 )); }) diff --git a/src/lib.rs b/src/lib.rs index ea04c22..6ccc1d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ //! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); //! //! // Python -> Rust -//! let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); +//! let new_sample: Sample = depythonize(&obj.into_bound(py)).unwrap(); //! //! assert_eq!(new_sample, sample); //! }); @@ -40,9 +40,9 @@ mod de; mod error; mod ser; +pub use crate::de::{depythonize, Depythonizer}; #[allow(deprecated)] -pub use crate::de::depythonize; -pub use crate::de::{depythonize_bound, Depythonizer}; +pub use crate::de::{depythonize_bound, depythonize_object}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ pythonize, pythonize_custom, PythonizeDefault, PythonizeDictType, PythonizeListType, diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index bcadabd..09e82fe 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,8 +6,7 @@ use pyo3::{ types::{PyDict, PyList, PyMapping, PySequence}, }; use pythonize::{ - depythonize_bound, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, - Pythonizer, + depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -70,7 +69,7 @@ fn test_custom_list() { .into_bound(py); assert!(serialized.is_instance_of::()); - let deserialized: Value = depythonize_bound(serialized).unwrap(); + let deserialized: Value = depythonize(&serialized).unwrap(); assert_eq!(deserialized, json!([1, 2, 3])); }) } @@ -135,7 +134,7 @@ fn test_custom_dict() { .into_bound(py); assert!(serialized.is_instance_of::()); - let deserialized: Value = depythonize_bound(serialized).unwrap(); + let deserialized: Value = depythonize(&serialized).unwrap(); assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); }) } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 6b2c4aa..1239913 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -54,7 +54,7 @@ fn test_de_valid() { pyroot.set_item("root_map", nested).unwrap(); - let de = &mut pythonize::Depythonizer::from_object_bound(pyroot.into_any()); + let de = &mut pythonize::Depythonizer::from_object(&pyroot); let root: Root = serde_path_to_error::deserialize(de).unwrap(); assert_eq!( @@ -96,7 +96,7 @@ fn test_de_invalid() { pyroot.set_item("root_map", nested).unwrap(); - let de = &mut pythonize::Depythonizer::from_object_bound(pyroot.into_any()); + let de = &mut pythonize::Depythonizer::from_object(&pyroot); let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); From 87dca55bb9f0140388b6ed2376ee14a2dc957a50 Mon Sep 17 00:00:00 2001 From: Henning Holm Date: Tue, 25 Jun 2024 21:09:48 +0200 Subject: [PATCH 29/77] Update to PyO3 0.22 --- CHANGELOG.md | 4 ++++ Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ebb355..04d3ee7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Unreleased + +- Update to PyO3 0.22 + ## 0.21.1 - 2024-04-02 - Fix compile error when using PyO3 `abi3` feature targeting a minimum version below 3.10 diff --git a/Cargo.toml b/Cargo.toml index 79b8a91..23f47d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.21.0", default-features = false } +pyo3 = { version = "0.22.0", default-features = false, features = ["gil-refs"] } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.21.1", default-features = false, features = ["auto-initialize", "macros"] } +pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "gil-refs", "macros", "py-clone"] } serde_json = "1.0" maplit = "1.0.2" serde_path_to_error = "0.1.15" From c52204c1fcf7a719f424a70efe9992d55c8e2553 Mon Sep 17 00:00:00 2001 From: Henning Holm Date: Tue, 25 Jun 2024 22:33:32 +0200 Subject: [PATCH 30/77] Remove intermittent `gil-refs` feature use This removes all code that relies on deprecated PyO3 functionality that has now been moved behind the `gil-refs` feature and removes intermittent use of the feature. --- CHANGELOG.md | 3 +++ Cargo.toml | 4 ++-- src/de.rs | 24 +----------------------- src/error.rs | 11 +---------- src/lib.rs | 10 ++++------ 5 files changed, 11 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d3ee7..42fddde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## Unreleased - Update to PyO3 0.22 +- Remove deprecated `depythonize`, use `depythonize_bound` instead +- Remove deprecated `from_object`, use `from_object_bound` instead +- Remove conversion from `PyDowncastError` to `PythonizeError` ## 0.21.1 - 2024-04-02 diff --git a/Cargo.toml b/Cargo.toml index 23f47d2..519006b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.22.0", default-features = false, features = ["gil-refs"] } +pyo3 = { version = "0.22.0", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "gil-refs", "macros", "py-clone"] } +pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" maplit = "1.0.2" serde_path_to_error = "0.1.15" diff --git a/src/de.rs b/src/de.rs index c7174fd..77ded82 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,22 +1,9 @@ -use pyo3::{types::*, Bound, PyNativeType}; +use pyo3::{types::*, Bound}; use serde::de::{self, IntoDeserializer}; use serde::Deserialize; use crate::error::{PythonizeError, Result}; -/// Attempt to convert a Python object to an instance of `T` -#[deprecated( - since = "0.21.0", - note = "will be replaced by `depythonize_bound` in a future release" -)] -pub fn depythonize<'de, T>(obj: &'de PyAny) -> Result -where - T: Deserialize<'de>, -{ - let mut depythonizer = Depythonizer::from_object_bound(obj.as_borrowed().to_owned()); - T::deserialize(&mut depythonizer) -} - /// Attempt to convert a Python object to an instance of `T` pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result where @@ -32,15 +19,6 @@ pub struct Depythonizer<'py> { } impl<'py> Depythonizer<'py> { - /// Create a deserializer from a Python object - #[deprecated( - since = "0.21.0", - note = "will be replaced by `Depythonizer::from_object_bound` in a future version" - )] - pub fn from_object(input: &'py PyAny) -> Self { - Self::from_object_bound(input.as_borrowed().to_owned()) - } - /// Create a deserializer from a Python object pub fn from_object_bound(input: Bound<'py, PyAny>) -> Self { Depythonizer { input } diff --git a/src/error.rs b/src/error.rs index d3ddd71..4aee7ea 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,5 @@ +use pyo3::PyErr; use pyo3::{exceptions::*, DowncastError, DowncastIntoError}; -use pyo3::{PyDowncastError, PyErr}; use serde::{de, ser}; use std::error; use std::fmt::{self, Debug, Display}; @@ -145,15 +145,6 @@ impl From for PythonizeError { } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'a> From> for PythonizeError { - fn from(other: PyDowncastError) -> Self { - Self { - inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), - } - } -} - /// Handle errors that occur when attempting to use `PyAny::cast_as` impl<'a, 'py> From> for PythonizeError { fn from(other: DowncastError<'a, 'py>) -> Self { diff --git a/src/lib.rs b/src/lib.rs index ea04c22..fd5dc44 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,8 @@ //! # Examples //! ``` //! use serde::{Serialize, Deserialize}; -//! use pyo3::Python; -//! use pythonize::{depythonize, pythonize}; +//! use pyo3::{types::PyAnyMethods, Python}; +//! use pythonize::{depythonize_bound, pythonize}; //! //! #[derive(Debug, Serialize, Deserialize, PartialEq)] //! struct Sample { @@ -27,10 +27,10 @@ //! // Rust -> Python //! let obj = pythonize(py, &sample).unwrap(); //! -//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); +//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.bind(py).repr().unwrap())); //! //! // Python -> Rust -//! let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); +//! let new_sample: Sample = depythonize_bound(obj.into_bound(py)).unwrap(); //! //! assert_eq!(new_sample, sample); //! }); @@ -40,8 +40,6 @@ mod de; mod error; mod ser; -#[allow(deprecated)] -pub use crate::de::depythonize; pub use crate::de::{depythonize_bound, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ From dd0beb93688741c66b7e1d5717437788c29e5aca Mon Sep 17 00:00:00 2001 From: bit-web24 Date: Fri, 19 Jul 2024 01:17:05 +0530 Subject: [PATCH 31/77] test_nested_struct --- src/ser.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/ser.rs b/src/ser.rs index ba7fb30..a7edf0b 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -533,6 +533,30 @@ mod test { ); } + #[test] + fn test_nested_struct() { + #[derive(Serialize)] + struct Foo { + name: String, + bar: Bar, + } + + #[derive(Serialize)] + struct Bar { + name: String, + } + + test_ser( + Foo { + name: "foo".to_string(), + bar: Bar { + name: "bar".to_string(), + }, + }, + r#"{"name":"foo","bar":{"name":"bar"}}"#, + ) + } + #[test] fn test_tuple_struct() { #[derive(Serialize)] From ce46786c6e24ee200284da2d7c3287dbdbf118db Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:24:33 +0100 Subject: [PATCH 32/77] fix large integer handling --- CHANGELOG.md | 2 ++ src/de.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9356e5..68ba131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Support `u128` / `i128` integers. +- Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` - `depythonize()` now take a `&Bound` and is no longer depreciate - `depythonize_object()` replace the old `depythonize()` and is depreciated - `depythonize_bound()` is depreciated diff --git a/src/de.rs b/src/de.rs index 62bb7cb..5aa4b34 100644 --- a/src/de.rs +++ b/src/de.rs @@ -71,6 +71,38 @@ impl<'py, 'bound> Depythonizer<'py, 'bound> { fn dict_access(&self) -> Result> { PyMappingAccess::new(self.input.downcast()?) } + + fn deserialize_any_int<'de, V>(&self, int: &Bound<'_, PyInt>, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + if let Ok(x) = int.extract::() { + if let Ok(x) = u8::try_from(x) { + visitor.visit_u8(x) + } else if let Ok(x) = u16::try_from(x) { + visitor.visit_u16(x) + } else if let Ok(x) = u32::try_from(x) { + visitor.visit_u32(x) + } else if let Ok(x) = u64::try_from(x) { + visitor.visit_u64(x) + } else { + visitor.visit_u128(x) + } + } else { + let x: i128 = int.extract()?; + if let Ok(x) = i8::try_from(x) { + visitor.visit_i8(x) + } else if let Ok(x) = i16::try_from(x) { + visitor.visit_i16(x) + } else if let Ok(x) = i32::try_from(x) { + visitor.visit_i32(x) + } else if let Ok(x) = i64::try_from(x) { + visitor.visit_i64(x) + } else { + visitor.visit_i128(x) + } + } + } } macro_rules! deserialize_type { @@ -99,8 +131,8 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' self.deserialize_unit(visitor) } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if obj.is_instance_of::() { - self.deserialize_i64(visitor) + } else if let Ok(x) = obj.downcast::() { + self.deserialize_any_int(x, visitor) } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.is_instance_of::() { @@ -151,10 +183,12 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' deserialize_type!(deserialize_i16 => visit_i16); deserialize_type!(deserialize_i32 => visit_i32); deserialize_type!(deserialize_i64 => visit_i64); + deserialize_type!(deserialize_i128 => visit_i128); deserialize_type!(deserialize_u8 => visit_u8); deserialize_type!(deserialize_u16 => visit_u16); deserialize_type!(deserialize_u32 => visit_u32); deserialize_type!(deserialize_u64 => visit_u64); + deserialize_type!(deserialize_u128 => visit_u128); deserialize_type!(deserialize_f32 => visit_f32); deserialize_type!(deserialize_f64 => visit_f64); @@ -459,7 +493,7 @@ mod test { use super::*; use crate::error::ErrorImpl; use maplit::hashmap; - use pyo3::Python; + use pyo3::{IntoPy, Python}; use serde_json::{json, Value as JsonValue}; fn test_de(code: &str, expected: &T, expected_json: &JsonValue) @@ -749,4 +783,21 @@ mod test { let code = "{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"; test_de(code, &expected, &expected_json); } + + #[test] + fn test_int_limits() { + Python::with_gil(|py| { + // serde_json::Value supports u64 and i64 as maxiumum sizes + let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i64::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: u128 = depythonize(&u128::MAX.into_py(py).into_bound(py)).unwrap(); + let _: i128 = depythonize(&u128::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: i128 = depythonize(&i128::MAX.into_py(py).into_bound(py)).unwrap(); + let _: i128 = depythonize(&i128::MIN.into_py(py).into_bound(py)).unwrap(); + }); + } } From 5996e628652e25f2d55ccee018358e39589c2ef5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:32:02 +0100 Subject: [PATCH 33/77] fix macos workflows --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ec1f92..2c7c6d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,9 +36,10 @@ jobs: strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: + python-architecture: ["x64"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: [ - "macos-latest", + "macos-13", "ubuntu-latest", "windows-latest", ] @@ -47,6 +48,10 @@ jobs: - python-version: "3.12" os: "ubuntu-latest" rust: "1.56" + - python-version: "3.12" + python-architecture: "arm64" + os: "macos-latest" + rust: "stable" steps: - uses: actions/checkout@v4 @@ -55,7 +60,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: ${{ matrix.python-python-architecture }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master From 61192c03da3fa819ff4594f5b18dde9e7850bd2c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:39:18 +0100 Subject: [PATCH 34/77] extend coverage --- src/de.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/de.rs b/src/de.rs index 5aa4b34..b22bbdd 100644 --- a/src/de.rs +++ b/src/de.rs @@ -788,6 +788,21 @@ mod test { fn test_int_limits() { Python::with_gil(|py| { // serde_json::Value supports u64 and i64 as maxiumum sizes + let _: serde_json::Value = depythonize(&u8::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u8::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i8::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i8::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: serde_json::Value = depythonize(&u16::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u16::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i16::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i16::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: serde_json::Value = depythonize(&u32::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u32::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i32::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i32::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap(); From ae640add6252ba95e8006db334c52c3095afa72b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:56:57 +0100 Subject: [PATCH 35/77] bump MSRV to 1.63 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 1 + Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ec1f92..6a4136a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: include: - python-version: "3.12" os: "ubuntu-latest" - rust: "1.56" + rust: "1.63" steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3368d..674d96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- Bump MSRV to 1.63 - Update to PyO3 0.22 - Remove deprecated `from_object`, use `from_object_bound` instead - Remove conversion from `PyDowncastError` to `PythonizeError` diff --git a/Cargo.toml b/Cargo.toml index 519006b..b515ddf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "pythonize" version = "0.21.1" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" -rust-version = "1.56" +rust-version = "1.63" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" From 7fde371577b27bcbf21c1a8667d94bb1c97b9adf Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:58:15 +0100 Subject: [PATCH 36/77] adjust CHANGELOG --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 674d96d..394f3cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,7 @@ - Bump MSRV to 1.63 - Update to PyO3 0.22 -- Remove deprecated `from_object`, use `from_object_bound` instead -- Remove conversion from `PyDowncastError` to `PythonizeError` +- Remove support for PyO3's `gil-refs` feature - `depythonize()` now take a `&Bound` and is no longer deprecated - `depythonize_bound()` is now deprecated - `Depythonizer` now contains a `&Bound` and so has an extra lifetime `'bound` From e7cf258b1d608a412f847305da513f23b59e81d7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 21:58:31 +0100 Subject: [PATCH 37/77] additional coverage --- Cargo.toml | 1 + src/ser.rs | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b515ddf..645ca32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,5 +19,6 @@ pyo3 = { version = "0.22.0", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" +serde_bytes = "0.11" maplit = "1.0.2" serde_path_to_error = "0.1.15" diff --git a/src/ser.rs b/src/ser.rs index c70a820..282c011 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -483,7 +483,7 @@ mod test { use maplit::hashmap; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; - use pyo3::types::PyDict; + use pyo3::types::{PyBytes, PyDict}; use serde::Serialize; fn test_ser(src: T, expected: &str) @@ -659,6 +659,27 @@ mod test { ) } + #[test] + fn test_floats() { + #[derive(Serialize)] + struct Floats { + a: f32, + b: f64, + } + + test_ser(Floats { a: 1.0, b: 2.0 }, r#"{"a":1.0,"b":2.0}"#) + } + + #[test] + fn test_char() { + #[derive(Serialize)] + struct Char { + a: char, + } + + test_ser(Char { a: 'a' }, r#"{"a":"a"}"#) + } + #[test] fn test_bool() { test_ser(true, "true"); @@ -672,10 +693,21 @@ mod test { test_ser((), "null"); test_ser(S, "null"); + + test_ser(Some(1), "1"); + test_ser(None::, "null"); } #[test] fn test_bytes() { + // serde treats &[u8] as a sequence of integers due to lack of specialization test_ser(b"foo", "[102,111,111]"); + + Python::with_gil(|py| { + assert!(pythonize(py, serde_bytes::Bytes::new(b"foo")) + .expect("bytes will always serialize successfully") + .eq(&PyBytes::new_bound(py, b"foo")) + .expect("bytes will always compare successfully")); + }); } } From 96ae2e285ae36038ccf0770506e5d03163df572a Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sat, 18 May 2024 04:46:52 +0000 Subject: [PATCH 38/77] Implement PythonizeListType for PyTuple --- src/ser.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ser.rs b/src/ser.rs index 282c011..94f69a3 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -54,6 +54,19 @@ impl PythonizeListType for PyList { } } +impl PythonizeListType for PyTuple { + fn create_sequence( + py: Python, + elements: impl IntoIterator, + ) -> PyResult> + where + T: ToPyObject, + U: ExactSizeIterator, + { + Ok(PyTuple::new_bound(py, elements).into_sequence()) + } +} + pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { From 6cd708b70a9de17add7627a98655ea97a1eddcf9 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sat, 18 May 2024 04:47:56 +0000 Subject: [PATCH 39/77] Add support for prebuilding (named) mappings during serialisation --- src/ser.rs | 148 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 109 insertions(+), 39 deletions(-) diff --git a/src/ser.rs b/src/ser.rs index 94f69a3..d5cbab3 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,6 +1,9 @@ use std::marker::PhantomData; -use pyo3::types::{PyAnyMethods, PyDict, PyList, PyMapping, PySequence, PyString, PyTuple}; +use pyo3::types::{ + IntoPyDict, PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PySequence, PyString, + PyTuple, PyTupleMethods, +}; use pyo3::{Bound, IntoPy, PyAny, PyResult, Python, ToPyObject}; use serde::{ser, Serialize}; @@ -10,6 +13,39 @@ use crate::error::{PythonizeError, Result}; pub trait PythonizeDictType { /// Constructor fn create_mapping(py: Python) -> PyResult>; + + /// Constructor + fn create_mapping_with_items< + K: ToPyObject, + V: ToPyObject, + U: ExactSizeIterator, + >( + py: Python, + items: impl IntoIterator, + ) -> PyResult> { + let mapping = Self::create_mapping(py)?; + + for (key, value) in items { + mapping.set_item(key, value)?; + } + + Ok(mapping) + } + + /// Constructor, allows the mappings to be named + fn create_mapping_with_items_name< + 'py, + K: ToPyObject, + V: ToPyObject, + U: ExactSizeIterator, + >( + py: Python<'py>, + name: &str, + items: impl IntoIterator, + ) -> PyResult> { + let _name = name; + Self::create_mapping_with_items(py, items) + } } /// Trait for types which can represent a Python sequence @@ -36,6 +72,17 @@ impl PythonizeDictType for PyDict { fn create_mapping(py: Python) -> PyResult> { Ok(PyDict::new_bound(py).into_any().downcast_into().unwrap()) } + + fn create_mapping_with_items< + K: ToPyObject, + V: ToPyObject, + U: ExactSizeIterator, + >( + py: Python, + items: impl IntoIterator, + ) -> PyResult> { + Ok(items.into_py_dict_bound(py).into_mapping()) + } } impl PythonizeListType for PyList { @@ -129,27 +176,30 @@ pub struct PythonCollectionSerializer<'py, P> { #[doc(hidden)] pub struct PythonTupleVariantSerializer<'py, P> { + name: &'static str, variant: &'static str, inner: PythonCollectionSerializer<'py, P>, } #[doc(hidden)] pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { + name: &'static str, variant: &'static str, - inner: PythonDictSerializer<'py, P>, + inner: PythonStructDictSerializer<'py, P>, } #[doc(hidden)] -pub struct PythonDictSerializer<'py, P: PythonizeTypes> { +pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - dict: Bound<'py, PyMapping>, + name: &'static str, + fields: Vec<(&'static str, Bound<'py, PyAny>)>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - map: Bound<'py, PyMapping>, + items: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, key: Option>, _types: PhantomData

, } @@ -162,7 +212,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { type SerializeTupleStruct = PythonCollectionSerializer<'py, P>; type SerializeTupleVariant = PythonTupleVariantSerializer<'py, P>; type SerializeMap = PythonMapSerializer<'py, P>; - type SerializeStruct = PythonDictSerializer<'py, P>; + type SerializeStruct = PythonStructDictSerializer<'py, P>; type SerializeStructVariant = PythonStructVariantSerializer<'py, P>; fn serialize_bool(self, v: bool) -> Result> { @@ -262,7 +312,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_newtype_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, value: &T, @@ -270,9 +320,12 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { where T: ?Sized + Serialize, { - let d = PyDict::new_bound(self.py); - d.set_item(variant, value.serialize(self)?)?; - Ok(d.into_any()) + let m = P::Map::create_mapping_with_items_name( + self.py, + name, + [(variant, value.serialize(self)?)], + )?; + Ok(m.into_any()) } fn serialize_seq(self, len: Option) -> Result> { @@ -305,18 +358,22 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_tuple_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, len: usize, ) -> Result> { let inner = self.serialize_tuple(len)?; - Ok(PythonTupleVariantSerializer { variant, inner }) + Ok(PythonTupleVariantSerializer { + name, + variant, + inner, + }) } - fn serialize_map(self, _len: Option) -> Result> { + fn serialize_map(self, len: Option) -> Result> { Ok(PythonMapSerializer { - map: P::Map::create_mapping(self.py)?, + items: Vec::with_capacity(len.unwrap_or(0)), key: None, py: self.py, _types: PhantomData, @@ -325,28 +382,31 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_struct( self, - _name: &'static str, - _len: usize, - ) -> Result> { - Ok(PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + name: &'static str, + len: usize, + ) -> Result> { + Ok(PythonStructDictSerializer { py: self.py, + name, + fields: Vec::with_capacity(len), _types: PhantomData, }) } fn serialize_struct_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, - _len: usize, + len: usize, ) -> Result> { Ok(PythonStructVariantSerializer { + name, variant, - inner: PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + inner: PythonStructDictSerializer { py: self.py, + name: variant, + fields: Vec::with_capacity(len), _types: PhantomData, }, }) @@ -415,9 +475,12 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe } fn end(self) -> Result> { - let d = PyDict::new_bound(self.inner.py); - d.set_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; - Ok(d.into_any()) + let m = P::Map::create_mapping_with_items_name( + self.inner.py, + self.name, + [(self.variant, ser::SerializeTuple::end(self.inner)?)], + )?; + Ok(m.into_any()) } } @@ -437,21 +500,22 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { where T: ?Sized + Serialize, { - self.map.set_item( + self.items.push(( self.key .take() .expect("serialize_value should always be called after serialize_key"), pythonize_custom::(self.py, value)?, - )?; + )); Ok(()) } fn end(self) -> Result> { - Ok(self.map.into_any()) + let m = P::Map::create_mapping_with_items(self.py, self.items)?; + Ok(m.into_any()) } } -impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonDictSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -459,13 +523,14 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonDictSerializer<'py, where T: ?Sized + Serialize, { - Ok(self - .dict - .set_item(key, pythonize_custom::(self.py, value)?)?) + self.fields + .push((key, pythonize_custom::(self.py, value)?)); + Ok(()) } fn end(self) -> Result> { - Ok(self.dict.into_any()) + let m = P::Map::create_mapping_with_items_name(self.py, self.name, self.fields)?; + Ok(m.into_any()) } } @@ -478,15 +543,20 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant T: ?Sized + Serialize, { self.inner - .dict - .set_item(key, pythonize_custom::(self.inner.py, value)?)?; + .fields + .push((key, pythonize_custom::(self.inner.py, value)?)); Ok(()) } fn end(self) -> Result> { - let d = PyDict::new_bound(self.inner.py); - d.set_item(self.variant, self.inner.dict)?; - Ok(d.into_any()) + let v = P::Map::create_mapping_with_items_name( + self.inner.py, + self.inner.name, + self.inner.fields, + )?; + let m = + P::Map::create_mapping_with_items_name(self.inner.py, self.name, [(self.variant, v)])?; + Ok(m.into_any()) } } From 28f7becbf9a0f3c7593b800f8519e6a14b7cf6e5 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 29 Mar 2024 14:02:43 +0000 Subject: [PATCH 40/77] Allow deserializing an enum from any mapping --- src/de.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/de.rs b/src/de.rs index 11d0fcc..986d0a6 100644 --- a/src/de.rs +++ b/src/de.rs @@ -294,21 +294,21 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' V: de::Visitor<'de>, { let item = &self.input; - if let Ok(d) = item.downcast::() { - // Get the enum variant from the dict key - if d.len() != 1 { + if let Ok(s) = item.downcast::() { + visitor.visit_enum(s.to_cow()?.into_deserializer()) + } else if let Ok(m) = item.downcast::() { + // Get the enum variant from the mapping key + if m.len()? != 1 { return Err(PythonizeError::invalid_length_enum()); } - let variant = d - .keys() + let variant: Bound = m + .keys()? .get_item(0)? .downcast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; - let value = d.get_item(&variant)?.unwrap(); + let value = m.get_item(&variant)?; let mut de = Depythonizer::from_object(&value); visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) - } else if let Ok(s) = item.downcast::() { - visitor.visit_enum(s.to_cow()?.into_deserializer()) } else { Err(PythonizeError::invalid_enum_type()) } From cc2015edb9bc965edcab142f51ef735f43f3ae8d Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 04:55:25 +0000 Subject: [PATCH 41/77] Implement suggested API changes --- src/lib.rs | 4 +- src/ser.rs | 160 +++++++++++++-------------- tests/test_custom_types.rs | 50 +++++++-- tests/test_with_serde_path_to_err.rs | 1 + 4 files changed, 120 insertions(+), 95 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f85a025..98fdfab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,6 @@ pub use crate::de::depythonize_bound; pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, PythonizeDefault, PythonizeDictType, PythonizeListType, - PythonizeTypes, Pythonizer, + pythonize, pythonize_custom, MappingBuilder, PythonizeDefault, PythonizeListType, + PythonizeMappingType, PythonizeNamedMappingType, PythonizeTypes, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index d5cbab3..40379ce 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,8 +1,8 @@ use std::marker::PhantomData; use pyo3::types::{ - IntoPyDict, PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PySequence, PyString, - PyTuple, PyTupleMethods, + PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PySequence, PyString, PyTuple, + PyTupleMethods, }; use pyo3::{Bound, IntoPy, PyAny, PyResult, Python, ToPyObject}; use serde::{ser, Serialize}; @@ -10,42 +10,31 @@ use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; /// Trait for types which can represent a Python mapping -pub trait PythonizeDictType { - /// Constructor - fn create_mapping(py: Python) -> PyResult>; +pub trait PythonizeMappingType { + /// Builder type for Python mappings + type Builder<'py>: MappingBuilder<'py>; - /// Constructor - fn create_mapping_with_items< - K: ToPyObject, - V: ToPyObject, - U: ExactSizeIterator, - >( - py: Python, - items: impl IntoIterator, - ) -> PyResult> { - let mapping = Self::create_mapping(py)?; + /// Create a builder for a Python mapping + fn create_builder(py: Python, len: Option) -> PyResult>; +} - for (key, value) in items { - mapping.set_item(key, value)?; - } +/// Trait for types which can represent a Python mapping and have a name +pub trait PythonizeNamedMappingType { + /// Builder type for Python mappings with a name + type Builder<'py>: MappingBuilder<'py>; - Ok(mapping) - } + /// Create a builder for a Python mapping with a name + fn create_builder<'py>(py: Python<'py>, len: usize, name: &str) + -> PyResult>; +} - /// Constructor, allows the mappings to be named - fn create_mapping_with_items_name< - 'py, - K: ToPyObject, - V: ToPyObject, - U: ExactSizeIterator, - >( - py: Python<'py>, - name: &str, - items: impl IntoIterator, - ) -> PyResult> { - let _name = name; - Self::create_mapping_with_items(py, items) - } +/// Trait for types which can build a Python mapping +pub trait MappingBuilder<'py> { + /// Adds the key-value item to the mapping being built + fn push_item(&mut self, key: K, value: V) -> PyResult<()>; + + /// Build the Python mapping + fn finish(self) -> PyResult>; } /// Trait for types which can represent a Python sequence @@ -63,25 +52,40 @@ pub trait PythonizeListType: Sized { /// Custom types for serialization pub trait PythonizeTypes { /// Python map type (should be representable as python mapping) - type Map: PythonizeDictType; + type Map: PythonizeMappingType; + /// Python (struct-like) named map type (should be representable as python mapping) + type NamedMap: PythonizeNamedMappingType; /// Python sequence type (should be representable as python sequence) type List: PythonizeListType; } -impl PythonizeDictType for PyDict { - fn create_mapping(py: Python) -> PyResult> { - Ok(PyDict::new_bound(py).into_any().downcast_into().unwrap()) +impl PythonizeMappingType for PyDict { + type Builder<'py> = Bound<'py, Self>; + + fn create_builder(py: Python, _len: Option) -> PyResult> { + Ok(Self::new_bound(py)) } +} - fn create_mapping_with_items< - K: ToPyObject, - V: ToPyObject, - U: ExactSizeIterator, - >( - py: Python, - items: impl IntoIterator, - ) -> PyResult> { - Ok(items.into_py_dict_bound(py).into_mapping()) +impl PythonizeNamedMappingType for PyDict { + type Builder<'py> = Bound<'py, Self>; + + fn create_builder<'py>( + py: Python<'py>, + _len: usize, + _name: &str, + ) -> PyResult> { + Ok(Self::new_bound(py)) + } +} + +impl<'py> MappingBuilder<'py> for Bound<'py, PyDict> { + fn push_item(&mut self, key: K, value: V) -> PyResult<()> { + self.set_item(key, value) + } + + fn finish(self) -> PyResult> { + Ok(self.into_any().downcast_into().unwrap()) } } @@ -118,6 +122,7 @@ pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { type Map = PyDict; + type NamedMap = PyDict; type List = PyList; } @@ -191,15 +196,14 @@ pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { #[doc(hidden)] pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - name: &'static str, - fields: Vec<(&'static str, Bound<'py, PyAny>)>, + builder: ::Builder<'py>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - items: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, + builder: ::Builder<'py>, key: Option>, _types: PhantomData

, } @@ -320,12 +324,9 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { where T: ?Sized + Serialize, { - let m = P::Map::create_mapping_with_items_name( - self.py, - name, - [(variant, value.serialize(self)?)], - )?; - Ok(m.into_any()) + let mut m = P::NamedMap::create_builder(self.py, 1, name)?; + m.push_item(variant, value.serialize(self)?)?; + Ok(m.finish()?.into_any()) } fn serialize_seq(self, len: Option) -> Result> { @@ -373,7 +374,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_map(self, len: Option) -> Result> { Ok(PythonMapSerializer { - items: Vec::with_capacity(len.unwrap_or(0)), + builder: P::Map::create_builder(self.py, len)?, key: None, py: self.py, _types: PhantomData, @@ -387,8 +388,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { ) -> Result> { Ok(PythonStructDictSerializer { py: self.py, - name, - fields: Vec::with_capacity(len), + builder: P::NamedMap::create_builder(self.py, len, name)?, _types: PhantomData, }) } @@ -405,8 +405,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { variant, inner: PythonStructDictSerializer { py: self.py, - name: variant, - fields: Vec::with_capacity(len), + builder: P::NamedMap::create_builder(self.py, len, variant)?, _types: PhantomData, }, }) @@ -475,12 +474,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe } fn end(self) -> Result> { - let m = P::Map::create_mapping_with_items_name( - self.inner.py, - self.name, - [(self.variant, ser::SerializeTuple::end(self.inner)?)], - )?; - Ok(m.into_any()) + let mut m = P::NamedMap::create_builder(self.inner.py, 1, self.name)?; + m.push_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; + Ok(m.finish()?.into_any()) } } @@ -500,18 +496,17 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { where T: ?Sized + Serialize, { - self.items.push(( + self.builder.push_item( self.key .take() .expect("serialize_value should always be called after serialize_key"), pythonize_custom::(self.py, value)?, - )); + )?; Ok(()) } fn end(self) -> Result> { - let m = P::Map::create_mapping_with_items(self.py, self.items)?; - Ok(m.into_any()) + Ok(self.builder.finish()?.into_any()) } } @@ -523,14 +518,13 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer where T: ?Sized + Serialize, { - self.fields - .push((key, pythonize_custom::(self.py, value)?)); + self.builder + .push_item(key, pythonize_custom::(self.py, value)?)?; Ok(()) } fn end(self) -> Result> { - let m = P::Map::create_mapping_with_items_name(self.py, self.name, self.fields)?; - Ok(m.into_any()) + Ok(self.builder.finish()?.into_any()) } } @@ -543,20 +537,16 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant T: ?Sized + Serialize, { self.inner - .fields - .push((key, pythonize_custom::(self.inner.py, value)?)); + .builder + .push_item(key, pythonize_custom::(self.inner.py, value)?)?; Ok(()) } fn end(self) -> Result> { - let v = P::Map::create_mapping_with_items_name( - self.inner.py, - self.inner.name, - self.inner.fields, - )?; - let m = - P::Map::create_mapping_with_items_name(self.inner.py, self.name, [(self.variant, v)])?; - Ok(m.into_any()) + let v = self.inner.builder.finish()?; + let mut m = P::NamedMap::create_builder(self.inner.py, 1, self.name)?; + m.push_item(self.variant, v)?; + Ok(m.finish()?.into_any()) } } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 9027c74..67a4c0c 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,7 +6,8 @@ use pyo3::{ types::{PyDict, PyList, PyMapping, PySequence}, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, + depythonize, pythonize_custom, MappingBuilder, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -57,6 +58,7 @@ impl PythonizeListType for CustomList { struct PythonizeCustomList; impl PythonizeTypes for PythonizeCustomList { type Map = PyDict; + type NamedMap = PyDict; type List = CustomList; } @@ -103,22 +105,54 @@ impl CustomDict { } } -impl PythonizeDictType for CustomDict { - fn create_mapping(py: Python) -> PyResult> { - let mapping = Bound::new( +impl PythonizeMappingType for CustomDict { + type Builder<'py> = CustomDictBuilder<'py>; + + fn create_builder<'py>(py: Python<'py>, len: Option) -> PyResult> { + Bound::new( py, CustomDict { - items: HashMap::new(), + items: HashMap::with_capacity(len.unwrap_or(0)), }, - )? - .into_any(); - Ok(unsafe { mapping.downcast_into_unchecked() }) + ) + .map(CustomDictBuilder) + } +} + +impl PythonizeNamedMappingType for CustomDict { + type Builder<'py> = CustomDictBuilder<'py>; + + fn create_builder<'py>( + py: Python<'py>, + len: usize, + _name: &str, + ) -> PyResult> { + Bound::new( + py, + CustomDict { + items: HashMap::with_capacity(len), + }, + ) + .map(CustomDictBuilder) + } +} + +struct CustomDictBuilder<'py>(Bound<'py, CustomDict>); + +impl<'py> MappingBuilder<'py> for CustomDictBuilder<'py> { + fn push_item(&mut self, key: K, value: V) -> PyResult<()> { + unsafe { self.0.downcast_unchecked::() }.set_item(key, value) + } + + fn finish(self) -> PyResult> { + Ok(unsafe { self.0.into_any().downcast_into_unchecked() }) } } struct PythonizeCustomDict; impl PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; + type NamedMap = CustomDict; type List = PyList; } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index acb34d7..03a1001 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -15,6 +15,7 @@ struct Root { impl PythonizeTypes for Root { type Map = PyDict; + type NamedMap = PyDict; type List = PyList; } From 5211ccfcf8a5e1dcd795ee1c26894d5038baf395 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 04:56:45 +0000 Subject: [PATCH 42/77] Add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f245889..e245de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - `Depythonizer` now contains a `&Bound` and so has an extra lifetime `'bound` - `Depythonizer::from_object()` now takes a `&Bound` and is no longer deprecated - Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` +- Support serializing struct-like types to named mappings using `PythonizeTypes::NamedMap` ## 0.21.1 - 2024-04-02 From 40140ac2012183330792d68342ebd3e6348206d0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 05:00:32 +0000 Subject: [PATCH 43/77] Use static names --- src/ser.rs | 9 ++++++--- tests/test_custom_types.rs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ser.rs b/src/ser.rs index 40379ce..d193d1d 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -24,8 +24,11 @@ pub trait PythonizeNamedMappingType { type Builder<'py>: MappingBuilder<'py>; /// Create a builder for a Python mapping with a name - fn create_builder<'py>(py: Python<'py>, len: usize, name: &str) - -> PyResult>; + fn create_builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult>; } /// Trait for types which can build a Python mapping @@ -73,7 +76,7 @@ impl PythonizeNamedMappingType for PyDict { fn create_builder<'py>( py: Python<'py>, _len: usize, - _name: &str, + _name: &'static str, ) -> PyResult> { Ok(Self::new_bound(py)) } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 67a4c0c..24b98c7 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -125,7 +125,7 @@ impl PythonizeNamedMappingType for CustomDict { fn create_builder<'py>( py: Python<'py>, len: usize, - _name: &str, + _name: &'static str, ) -> PyResult> { Bound::new( py, From f3ff8a924126e438698c24770de7f0d3f2d0ee16 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 10:31:49 +0000 Subject: [PATCH 44/77] Don't use GATs --- src/lib.rs | 4 +- src/ser.rs | 119 +++++++++++++++++---------- tests/test_custom_types.rs | 47 ++++------- tests/test_with_serde_path_to_err.rs | 4 +- 4 files changed, 94 insertions(+), 80 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 98fdfab..f50e5d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,6 @@ pub use crate::de::depythonize_bound; pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, MappingBuilder, PythonizeDefault, PythonizeListType, - PythonizeMappingType, PythonizeNamedMappingType, PythonizeTypes, Pythonizer, + pythonize, pythonize_custom, PythonizeDefault, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingWrapper, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index d193d1d..7f9ec2c 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -12,32 +12,43 @@ use crate::error::{PythonizeError, Result}; /// Trait for types which can represent a Python mapping pub trait PythonizeMappingType { /// Builder type for Python mappings - type Builder<'py>: MappingBuilder<'py>; + type Builder<'py>; /// Create a builder for a Python mapping - fn create_builder(py: Python, len: Option) -> PyResult>; + fn builder(py: Python, len: Option) -> PyResult>; + + /// Adds the key-value item to the mapping being built + fn push_item( + builder: &mut Self::Builder<'_>, + key: K, + value: V, + ) -> PyResult<()>; + + /// Build the Python mapping + fn finish(builder: Self::Builder<'_>) -> PyResult>; } /// Trait for types which can represent a Python mapping and have a name pub trait PythonizeNamedMappingType { /// Builder type for Python mappings with a name - type Builder<'py>: MappingBuilder<'py>; + type Builder<'py>; /// Create a builder for a Python mapping with a name - fn create_builder<'py>( + fn builder<'py>( py: Python<'py>, len: usize, name: &'static str, ) -> PyResult>; -} -/// Trait for types which can build a Python mapping -pub trait MappingBuilder<'py> { /// Adds the key-value item to the mapping being built - fn push_item(&mut self, key: K, value: V) -> PyResult<()>; + fn push_item( + builder: &mut Self::Builder<'_>, + key: K, + value: V, + ) -> PyResult<()>; /// Build the Python mapping - fn finish(self) -> PyResult>; + fn finish(builder: Self::Builder<'_>) -> PyResult>; } /// Trait for types which can represent a Python sequence @@ -65,30 +76,46 @@ pub trait PythonizeTypes { impl PythonizeMappingType for PyDict { type Builder<'py> = Bound<'py, Self>; - fn create_builder(py: Python, _len: Option) -> PyResult> { + fn builder(py: Python, _len: Option) -> PyResult> { Ok(Self::new_bound(py)) } + + fn push_item( + builder: &mut Self::Builder<'_>, + key: K, + value: V, + ) -> PyResult<()> { + builder.set_item(key, value) + } + + fn finish(builder: Self::Builder<'_>) -> PyResult> { + Ok(builder.into_any().downcast_into().unwrap()) + } } -impl PythonizeNamedMappingType for PyDict { - type Builder<'py> = Bound<'py, Self>; +pub struct PythonizeUnnamedMappingWrapper(T); + +impl PythonizeNamedMappingType for PythonizeUnnamedMappingWrapper { + type Builder<'py> = ::Builder<'py>; - fn create_builder<'py>( + fn builder<'py>( py: Python<'py>, - _len: usize, + len: usize, _name: &'static str, ) -> PyResult> { - Ok(Self::new_bound(py)) + ::builder(py, Some(len)) } -} -impl<'py> MappingBuilder<'py> for Bound<'py, PyDict> { - fn push_item(&mut self, key: K, value: V) -> PyResult<()> { - self.set_item(key, value) + fn push_item( + builder: &mut Self::Builder<'_>, + key: K, + value: V, + ) -> PyResult<()> { + ::push_item(builder, key, value) } - fn finish(self) -> PyResult> { - Ok(self.into_any().downcast_into().unwrap()) + fn finish(builder: Self::Builder<'_>) -> PyResult> { + ::finish(builder) } } @@ -125,7 +152,7 @@ pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { type Map = PyDict; - type NamedMap = PyDict; + type NamedMap = PythonizeUnnamedMappingWrapper; type List = PyList; } @@ -327,9 +354,9 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { where T: ?Sized + Serialize, { - let mut m = P::NamedMap::create_builder(self.py, 1, name)?; - m.push_item(variant, value.serialize(self)?)?; - Ok(m.finish()?.into_any()) + let mut m = P::NamedMap::builder(self.py, 1, name)?; + P::NamedMap::push_item(&mut m, variant, value.serialize(self)?)?; + Ok(P::NamedMap::finish(m)?.into_any()) } fn serialize_seq(self, len: Option) -> Result> { @@ -377,7 +404,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_map(self, len: Option) -> Result> { Ok(PythonMapSerializer { - builder: P::Map::create_builder(self.py, len)?, + builder: P::Map::builder(self.py, len)?, key: None, py: self.py, _types: PhantomData, @@ -391,7 +418,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { ) -> Result> { Ok(PythonStructDictSerializer { py: self.py, - builder: P::NamedMap::create_builder(self.py, len, name)?, + builder: P::NamedMap::builder(self.py, len, name)?, _types: PhantomData, }) } @@ -408,7 +435,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { variant, inner: PythonStructDictSerializer { py: self.py, - builder: P::NamedMap::create_builder(self.py, len, variant)?, + builder: P::NamedMap::builder(self.py, len, variant)?, _types: PhantomData, }, }) @@ -477,9 +504,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe } fn end(self) -> Result> { - let mut m = P::NamedMap::create_builder(self.inner.py, 1, self.name)?; - m.push_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; - Ok(m.finish()?.into_any()) + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_item(&mut m, self.variant, ser::SerializeTuple::end(self.inner)?)?; + Ok(P::NamedMap::finish(m)?.into_any()) } } @@ -499,7 +526,8 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { where T: ?Sized + Serialize, { - self.builder.push_item( + P::Map::push_item( + &mut self.builder, self.key .take() .expect("serialize_value should always be called after serialize_key"), @@ -509,7 +537,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { } fn end(self) -> Result> { - Ok(self.builder.finish()?.into_any()) + Ok(P::Map::finish(self.builder)?.into_any()) } } @@ -521,13 +549,16 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer where T: ?Sized + Serialize, { - self.builder - .push_item(key, pythonize_custom::(self.py, value)?)?; + P::NamedMap::push_item( + &mut self.builder, + key, + pythonize_custom::(self.py, value)?, + )?; Ok(()) } fn end(self) -> Result> { - Ok(self.builder.finish()?.into_any()) + Ok(P::NamedMap::finish(self.builder)?.into_any()) } } @@ -539,17 +570,19 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant where T: ?Sized + Serialize, { - self.inner - .builder - .push_item(key, pythonize_custom::(self.inner.py, value)?)?; + P::NamedMap::push_item( + &mut self.inner.builder, + key, + pythonize_custom::(self.inner.py, value)?, + )?; Ok(()) } fn end(self) -> Result> { - let v = self.inner.builder.finish()?; - let mut m = P::NamedMap::create_builder(self.inner.py, 1, self.name)?; - m.push_item(self.variant, v)?; - Ok(m.finish()?.into_any()) + let v = P::NamedMap::finish(self.inner.builder)?; + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_item(&mut m, self.variant, v)?; + Ok(P::NamedMap::finish(m)?.into_any()) } } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 24b98c7..77d8a12 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,8 +6,8 @@ use pyo3::{ types::{PyDict, PyList, PyMapping, PySequence}, }; use pythonize::{ - depythonize, pythonize_custom, MappingBuilder, PythonizeListType, PythonizeMappingType, - PythonizeNamedMappingType, PythonizeTypes, Pythonizer, + depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, PythonizeTypes, + PythonizeUnnamedMappingWrapper, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -58,7 +58,7 @@ impl PythonizeListType for CustomList { struct PythonizeCustomList; impl PythonizeTypes for PythonizeCustomList { type Map = PyDict; - type NamedMap = PyDict; + type NamedMap = PythonizeUnnamedMappingWrapper; type List = CustomList; } @@ -106,53 +106,34 @@ impl CustomDict { } impl PythonizeMappingType for CustomDict { - type Builder<'py> = CustomDictBuilder<'py>; + type Builder<'py> = Bound<'py, CustomDict>; - fn create_builder<'py>(py: Python<'py>, len: Option) -> PyResult> { + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult> { Bound::new( py, CustomDict { items: HashMap::with_capacity(len.unwrap_or(0)), }, ) - .map(CustomDictBuilder) } -} - -impl PythonizeNamedMappingType for CustomDict { - type Builder<'py> = CustomDictBuilder<'py>; - - fn create_builder<'py>( - py: Python<'py>, - len: usize, - _name: &'static str, - ) -> PyResult> { - Bound::new( - py, - CustomDict { - items: HashMap::with_capacity(len), - }, - ) - .map(CustomDictBuilder) - } -} - -struct CustomDictBuilder<'py>(Bound<'py, CustomDict>); -impl<'py> MappingBuilder<'py> for CustomDictBuilder<'py> { - fn push_item(&mut self, key: K, value: V) -> PyResult<()> { - unsafe { self.0.downcast_unchecked::() }.set_item(key, value) + fn push_item<'py, K: ToPyObject, V: ToPyObject>( + builder: &mut Self::Builder<'py>, + key: K, + value: V, + ) -> PyResult<()> { + unsafe { builder.downcast_unchecked::() }.set_item(key, value) } - fn finish(self) -> PyResult> { - Ok(unsafe { self.0.into_any().downcast_into_unchecked() }) + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(unsafe { builder.into_any().downcast_into_unchecked() }) } } struct PythonizeCustomDict; impl PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; - type NamedMap = CustomDict; + type NamedMap = PythonizeUnnamedMappingWrapper; type List = PyList; } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 03a1001..80dba74 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -4,7 +4,7 @@ use pyo3::{ prelude::*, types::{PyDict, PyList}, }; -use pythonize::PythonizeTypes; +use pythonize::{PythonizeTypes, PythonizeUnnamedMappingWrapper}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -15,7 +15,7 @@ struct Root { impl PythonizeTypes for Root { type Map = PyDict; - type NamedMap = PyDict; + type NamedMap = PythonizeUnnamedMappingWrapper; type List = PyList; } From aa8f2ab11944432d4f6f26b76c3b7cce8c1e6e80 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 10:35:50 +0000 Subject: [PATCH 45/77] Amend the CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e245de9..d6c1c1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - `Depythonizer` now contains a `&Bound` and so has an extra lifetime `'bound` - `Depythonizer::from_object()` now takes a `&Bound` and is no longer deprecated - Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` +- Implement `PythonizeListType` for `PyTuple` +- Support deserializing enums from any `PyMapping` instead of just `PyDict` - Support serializing struct-like types to named mappings using `PythonizeTypes::NamedMap` ## 0.21.1 - 2024-04-02 From dade974353530fc1c958cb136318a8cd11e4d725 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Sun, 4 Aug 2024 10:43:49 +0000 Subject: [PATCH 46/77] Add test for PythonizeListType impl for PyTuple --- tests/test_custom_types.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 77d8a12..f633a81 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::{ exceptions::{PyIndexError, PyKeyError}, prelude::*, - types::{PyDict, PyList, PyMapping, PySequence}, + types::{PyDict, PyMapping, PySequence, PyTuple}, }; use pythonize::{ depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, PythonizeTypes, @@ -134,7 +134,7 @@ struct PythonizeCustomDict; impl PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; type NamedMap = PythonizeUnnamedMappingWrapper; - type List = PyList; + type List = PyTuple; } #[test] @@ -151,6 +151,19 @@ fn test_custom_dict() { }) } +#[test] +fn test_tuple() { + Python::with_gil(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &json!([1, 2, 3, 4])).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!([1, 2, 3, 4])); + }) +} + #[test] fn test_pythonizer_can_be_created() { // https://github.com/davidhewitt/pythonize/pull/56 From 893a1bce8258a5c67b2283c5da5d58e4df9526ec Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:39:10 +0000 Subject: [PATCH 47/77] Apply code review comments --- src/ser.rs | 177 ++++++++++++++++----------- tests/test_custom_types.rs | 24 ++-- tests/test_with_serde_path_to_err.rs | 4 +- 3 files changed, 121 insertions(+), 84 deletions(-) diff --git a/src/ser.rs b/src/ser.rs index 7f9ec2c..c2cbfce 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -9,46 +9,44 @@ use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; +// TODO: move 'py lifetime into builder once GATs are available in MSRV /// Trait for types which can represent a Python mapping -pub trait PythonizeMappingType { +pub trait PythonizeMappingType<'py> { /// Builder type for Python mappings - type Builder<'py>; + type Builder; /// Create a builder for a Python mapping - fn builder(py: Python, len: Option) -> PyResult>; + fn builder(py: Python<'py>, len: Option) -> PyResult; /// Adds the key-value item to the mapping being built - fn push_item( - builder: &mut Self::Builder<'_>, - key: K, - value: V, + fn push_item( + builder: &mut Self::Builder, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, ) -> PyResult<()>; /// Build the Python mapping - fn finish(builder: Self::Builder<'_>) -> PyResult>; + fn finish(builder: Self::Builder) -> PyResult>; } +// TODO: move 'py lifetime into builder once GATs are available in MSRV /// Trait for types which can represent a Python mapping and have a name -pub trait PythonizeNamedMappingType { +pub trait PythonizeNamedMappingType<'py> { /// Builder type for Python mappings with a name - type Builder<'py>; + type Builder; /// Create a builder for a Python mapping with a name - fn builder<'py>( - py: Python<'py>, - len: usize, - name: &'static str, - ) -> PyResult>; + fn builder(py: Python<'py>, len: usize, name: &'static str) -> PyResult; - /// Adds the key-value item to the mapping being built - fn push_item( - builder: &mut Self::Builder<'_>, - key: K, - value: V, + /// Adds the field to the named mapping being built + fn push_field( + builder: &mut Self::Builder, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, ) -> PyResult<()>; /// Build the Python mapping - fn finish(builder: Self::Builder<'_>) -> PyResult>; + fn finish(builder: Self::Builder) -> PyResult>; } /// Trait for types which can represent a Python sequence @@ -63,58 +61,81 @@ pub trait PythonizeListType: Sized { U: ExactSizeIterator; } +// TODO: remove 'py lifetime once GATs are available in MSRV /// Custom types for serialization -pub trait PythonizeTypes { +pub trait PythonizeTypes<'py> { /// Python map type (should be representable as python mapping) - type Map: PythonizeMappingType; + type Map: PythonizeMappingType<'py>; /// Python (struct-like) named map type (should be representable as python mapping) - type NamedMap: PythonizeNamedMappingType; + type NamedMap: PythonizeNamedMappingType<'py>; /// Python sequence type (should be representable as python sequence) type List: PythonizeListType; } -impl PythonizeMappingType for PyDict { - type Builder<'py> = Bound<'py, Self>; +impl<'py> PythonizeMappingType<'py> for PyDict { + type Builder = Bound<'py, Self>; - fn builder(py: Python, _len: Option) -> PyResult> { + fn builder(py: Python<'py>, _len: Option) -> PyResult { Ok(Self::new_bound(py)) } - fn push_item( - builder: &mut Self::Builder<'_>, - key: K, - value: V, + fn push_item( + builder: &mut Self::Builder, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, ) -> PyResult<()> { builder.set_item(key, value) } - fn finish(builder: Self::Builder<'_>) -> PyResult> { - Ok(builder.into_any().downcast_into().unwrap()) + fn finish(builder: Self::Builder) -> PyResult> { + Ok(builder.into_mapping()) } } -pub struct PythonizeUnnamedMappingWrapper(T); +pub struct PythonizeUnnamedMappingWrapper<'py, T: PythonizeMappingType<'py>> { + unnamed: T, + _marker: PhantomData<&'py ()>, +} -impl PythonizeNamedMappingType for PythonizeUnnamedMappingWrapper { - type Builder<'py> = ::Builder<'py>; +impl<'py, T: PythonizeMappingType<'py>> PythonizeUnnamedMappingWrapper<'py, T> { + #[must_use] + pub fn new(unnamed: T) -> Self { + Self { + unnamed, + _marker: PhantomData::<&'py ()>, + } + } - fn builder<'py>( - py: Python<'py>, - len: usize, - _name: &'static str, - ) -> PyResult> { + #[must_use] + pub fn into_inner(self) -> T { + self.unnamed + } +} + +impl<'py, T: PythonizeMappingType<'py>> From for PythonizeUnnamedMappingWrapper<'py, T> { + fn from(value: T) -> Self { + Self::new(value) + } +} + +impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> + for PythonizeUnnamedMappingWrapper<'py, T> +{ + type Builder = >::Builder; + + fn builder(py: Python<'py>, len: usize, _name: &'static str) -> PyResult { ::builder(py, Some(len)) } - fn push_item( - builder: &mut Self::Builder<'_>, - key: K, - value: V, + fn push_field( + builder: &mut Self::Builder, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, ) -> PyResult<()> { - ::push_item(builder, key, value) + ::push_item(builder, name.into_any(), value) } - fn finish(builder: Self::Builder<'_>) -> PyResult> { + fn finish(builder: Self::Builder) -> PyResult> { ::finish(builder) } } @@ -150,9 +171,9 @@ impl PythonizeListType for PyTuple { pub struct PythonizeDefault; -impl PythonizeTypes for PythonizeDefault { +impl<'py> PythonizeTypes<'py> for PythonizeDefault { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper; + type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; type List = PyList; } @@ -169,7 +190,7 @@ where pub fn pythonize_custom<'py, P, T>(py: Python<'py>, value: &T) -> Result> where T: ?Sized + Serialize, - P: PythonizeTypes, + P: PythonizeTypes<'py>, { value.serialize(Pythonizer::custom::

(py)) } @@ -217,28 +238,28 @@ pub struct PythonTupleVariantSerializer<'py, P> { } #[doc(hidden)] -pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { +pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes<'py>> { name: &'static str, variant: &'static str, inner: PythonStructDictSerializer<'py, P>, } #[doc(hidden)] -pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { +pub struct PythonStructDictSerializer<'py, P: PythonizeTypes<'py>> { py: Python<'py>, - builder: ::Builder<'py>, + builder: >::Builder, _types: PhantomData

, } #[doc(hidden)] -pub struct PythonMapSerializer<'py, P: PythonizeTypes> { +pub struct PythonMapSerializer<'py, P: PythonizeTypes<'py>> { py: Python<'py>, - builder: ::Builder<'py>, + builder: >::Builder, key: Option>, _types: PhantomData

, } -impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; type SerializeSeq = PythonCollectionSerializer<'py, P>; @@ -355,7 +376,11 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { T: ?Sized + Serialize, { let mut m = P::NamedMap::builder(self.py, 1, name)?; - P::NamedMap::push_item(&mut m, variant, value.serialize(self)?)?; + P::NamedMap::push_field( + &mut m, + PyString::new_bound(self.py, variant), + value.serialize(self)?, + )?; Ok(P::NamedMap::finish(m)?.into_any()) } @@ -442,7 +467,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { } } -impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeSeq for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -460,7 +485,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'p } } -impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeTuple for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -476,7 +501,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer< } } -impl<'py, P: PythonizeTypes> ser::SerializeTupleStruct for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleStruct for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -492,7 +517,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleStruct for PythonCollectionSeria } } -impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleVariant + for PythonTupleVariantSerializer<'py, P> +{ type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -505,12 +532,16 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe fn end(self) -> Result> { let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; - P::NamedMap::push_item(&mut m, self.variant, ser::SerializeTuple::end(self.inner)?)?; + P::NamedMap::push_field( + &mut m, + PyString::new_bound(self.inner.py, self.variant), + ser::SerializeTuple::end(self.inner)?, + )?; Ok(P::NamedMap::finish(m)?.into_any()) } } -impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeMap for PythonMapSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -541,7 +572,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { } } -impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -549,9 +580,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer where T: ?Sized + Serialize, { - P::NamedMap::push_item( + P::NamedMap::push_field( &mut self.builder, - key, + PyString::new_bound(self.py, key), pythonize_custom::(self.py, value)?, )?; Ok(()) @@ -562,7 +593,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer } } -impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariantSerializer<'py, P> { +impl<'py, P: PythonizeTypes<'py>> ser::SerializeStructVariant + for PythonStructVariantSerializer<'py, P> +{ type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -570,9 +603,9 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant where T: ?Sized + Serialize, { - P::NamedMap::push_item( + P::NamedMap::push_field( &mut self.inner.builder, - key, + PyString::new_bound(self.inner.py, key), pythonize_custom::(self.inner.py, value)?, )?; Ok(()) @@ -581,7 +614,11 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant fn end(self) -> Result> { let v = P::NamedMap::finish(self.inner.builder)?; let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; - P::NamedMap::push_item(&mut m, self.variant, v)?; + P::NamedMap::push_field( + &mut m, + PyString::new_bound(self.inner.py, self.variant), + v.into_any(), + )?; Ok(P::NamedMap::finish(m)?.into_any()) } } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index f633a81..e347763 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -56,9 +56,9 @@ impl PythonizeListType for CustomList { } struct PythonizeCustomList; -impl PythonizeTypes for PythonizeCustomList { +impl<'py> PythonizeTypes<'py> for PythonizeCustomList { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper; + type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; type List = CustomList; } @@ -105,10 +105,10 @@ impl CustomDict { } } -impl PythonizeMappingType for CustomDict { - type Builder<'py> = Bound<'py, CustomDict>; +impl<'py> PythonizeMappingType<'py> for CustomDict { + type Builder = Bound<'py, CustomDict>; - fn builder<'py>(py: Python<'py>, len: Option) -> PyResult> { + fn builder(py: Python<'py>, len: Option) -> PyResult { Bound::new( py, CustomDict { @@ -117,23 +117,23 @@ impl PythonizeMappingType for CustomDict { ) } - fn push_item<'py, K: ToPyObject, V: ToPyObject>( - builder: &mut Self::Builder<'py>, - key: K, - value: V, + fn push_item( + builder: &mut Self::Builder, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, ) -> PyResult<()> { unsafe { builder.downcast_unchecked::() }.set_item(key, value) } - fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + fn finish(builder: Self::Builder) -> PyResult> { Ok(unsafe { builder.into_any().downcast_into_unchecked() }) } } struct PythonizeCustomDict; -impl PythonizeTypes for PythonizeCustomDict { +impl<'py> PythonizeTypes<'py> for PythonizeCustomDict { type Map = CustomDict; - type NamedMap = PythonizeUnnamedMappingWrapper; + type NamedMap = PythonizeUnnamedMappingWrapper<'py, CustomDict>; type List = PyTuple; } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 80dba74..cc624a0 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -13,9 +13,9 @@ struct Root { root_map: BTreeMap>, } -impl PythonizeTypes for Root { +impl<'py, T> PythonizeTypes<'py> for Root { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper; + type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; type List = PyList; } From 38d7a056bef4cea3c9f682ae98405d019ba5a81a Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:44:05 +0000 Subject: [PATCH 48/77] Add short docs for the unnamed mapping adapter type --- src/lib.rs | 2 +- src/ser.rs | 17 ++++++++++++----- tests/test_custom_types.rs | 6 +++--- tests/test_with_serde_path_to_err.rs | 4 ++-- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f50e5d5..4ce1751 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,5 +46,5 @@ pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ pythonize, pythonize_custom, PythonizeDefault, PythonizeListType, PythonizeMappingType, - PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingWrapper, Pythonizer, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index c2cbfce..f828584 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -92,12 +92,19 @@ impl<'py> PythonizeMappingType<'py> for PyDict { } } -pub struct PythonizeUnnamedMappingWrapper<'py, T: PythonizeMappingType<'py>> { +/// Adapter type to use an unnamed mapping type, i.e. one that implements +/// [`PythonizeMappingType`], as a named mapping type, i.e. one that implements +/// [`PythonizeNamedMappingType`]. The adapter simply drops the provided name. +/// +/// This adapter is commonly applied to use the same unnamed mapping type for +/// both [`PythonizeTypes::Map`] and [`PythonizeTypes::NamedMap`] while only +/// implementing [`PythonizeMappingType`]. +pub struct PythonizeUnnamedMappingAdapter<'py, T: PythonizeMappingType<'py>> { unnamed: T, _marker: PhantomData<&'py ()>, } -impl<'py, T: PythonizeMappingType<'py>> PythonizeUnnamedMappingWrapper<'py, T> { +impl<'py, T: PythonizeMappingType<'py>> PythonizeUnnamedMappingAdapter<'py, T> { #[must_use] pub fn new(unnamed: T) -> Self { Self { @@ -112,14 +119,14 @@ impl<'py, T: PythonizeMappingType<'py>> PythonizeUnnamedMappingWrapper<'py, T> { } } -impl<'py, T: PythonizeMappingType<'py>> From for PythonizeUnnamedMappingWrapper<'py, T> { +impl<'py, T: PythonizeMappingType<'py>> From for PythonizeUnnamedMappingAdapter<'py, T> { fn from(value: T) -> Self { Self::new(value) } } impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> - for PythonizeUnnamedMappingWrapper<'py, T> + for PythonizeUnnamedMappingAdapter<'py, T> { type Builder = >::Builder; @@ -173,7 +180,7 @@ pub struct PythonizeDefault; impl<'py> PythonizeTypes<'py> for PythonizeDefault { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; type List = PyList; } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index e347763..f5e450a 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -7,7 +7,7 @@ use pyo3::{ }; use pythonize::{ depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, PythonizeTypes, - PythonizeUnnamedMappingWrapper, Pythonizer, + PythonizeUnnamedMappingAdapter, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -58,7 +58,7 @@ impl PythonizeListType for CustomList { struct PythonizeCustomList; impl<'py> PythonizeTypes<'py> for PythonizeCustomList { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; type List = CustomList; } @@ -133,7 +133,7 @@ impl<'py> PythonizeMappingType<'py> for CustomDict { struct PythonizeCustomDict; impl<'py> PythonizeTypes<'py> for PythonizeCustomDict { type Map = CustomDict; - type NamedMap = PythonizeUnnamedMappingWrapper<'py, CustomDict>; + type NamedMap = PythonizeUnnamedMappingAdapter<'py, CustomDict>; type List = PyTuple; } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index cc624a0..1321c2b 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -4,7 +4,7 @@ use pyo3::{ prelude::*, types::{PyDict, PyList}, }; -use pythonize::{PythonizeTypes, PythonizeUnnamedMappingWrapper}; +use pythonize::{PythonizeTypes, PythonizeUnnamedMappingAdapter}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -15,7 +15,7 @@ struct Root { impl<'py, T> PythonizeTypes<'py> for Root { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingWrapper<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; type List = PyList; } From aa961a9bbbcde6e4897aed973039c6a6169a2f5c Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Tue, 6 Aug 2024 21:01:48 +0000 Subject: [PATCH 49/77] Add test for unnamed and named mappings --- tests/test_custom_types.rs | 103 ++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index f5e450a..5f0084b 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -6,8 +6,8 @@ use pyo3::{ types::{PyDict, PyMapping, PySequence, PyTuple}, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, PythonizeTypes, - PythonizeUnnamedMappingAdapter, Pythonizer, + depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; @@ -180,3 +180,102 @@ fn test_pythonizer_can_be_created() { .is_instance_of::()); }) } + +#[pyclass(mapping)] +struct NamedCustomDict { + name: String, + items: HashMap, +} + +#[pymethods] +impl NamedCustomDict { + fn __len__(&self) -> usize { + self.items.len() + } + + fn __getitem__(&self, key: String) -> PyResult { + self.items + .get(&key) + .cloned() + .ok_or_else(|| PyKeyError::new_err(key)) + } + + fn __setitem__(&mut self, key: String, value: PyObject) { + self.items.insert(key, value); + } + + fn keys(&self) -> Vec<&String> { + self.items.keys().collect() + } + + fn values(&self) -> Vec { + self.items.values().cloned().collect() + } +} + +impl<'py> PythonizeNamedMappingType<'py> for NamedCustomDict { + type Builder = Bound<'py, NamedCustomDict>; + + fn builder(py: Python<'py>, len: usize, name: &'static str) -> PyResult { + Bound::new( + py, + NamedCustomDict { + name: String::from(name), + items: HashMap::with_capacity(len), + }, + ) + } + + fn push_field( + builder: &mut Self::Builder, + name: Bound<'py, pyo3::types::PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + unsafe { builder.downcast_unchecked::() }.set_item(name, value) + } + + fn finish(builder: Self::Builder) -> PyResult> { + Ok(unsafe { builder.into_any().downcast_into_unchecked() }) + } +} + +struct PythonizeNamedCustomDict; +impl<'py> PythonizeTypes<'py> for PythonizeNamedCustomDict { + type Map = CustomDict; + type NamedMap = NamedCustomDict; + type List = PyTuple; +} + +#[derive(Serialize)] +struct Struct { + hello: u8, + world: i8, +} + +#[test] +fn test_custom_unnamed_dict() { + Python::with_gil(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} + +#[test] +fn test_custom_named_dict() { + Python::with_gil(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }) + .unwrap(); + let named: Bound = serialized.extract().unwrap(); + assert_eq!(named.borrow().name, "Struct"); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} From 13df99f1fea3986745215168e9f72e4419baf4fb Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:54:03 +0300 Subject: [PATCH 50/77] Remove constructor from type-level-only adapter The adapter is anyways constructed through the builder method Co-authored-by: David Hewitt --- src/ser.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/ser.rs b/src/ser.rs index f828584..ecb995b 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -104,26 +104,6 @@ pub struct PythonizeUnnamedMappingAdapter<'py, T: PythonizeMappingType<'py>> { _marker: PhantomData<&'py ()>, } -impl<'py, T: PythonizeMappingType<'py>> PythonizeUnnamedMappingAdapter<'py, T> { - #[must_use] - pub fn new(unnamed: T) -> Self { - Self { - unnamed, - _marker: PhantomData::<&'py ()>, - } - } - - #[must_use] - pub fn into_inner(self) -> T { - self.unnamed - } -} - -impl<'py, T: PythonizeMappingType<'py>> From for PythonizeUnnamedMappingAdapter<'py, T> { - fn from(value: T) -> Self { - Self::new(value) - } -} impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> for PythonizeUnnamedMappingAdapter<'py, T> From 7c18b9f398287f7689dfdd41f75de23d8194e1a6 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:13:45 +0000 Subject: [PATCH 51/77] Fix fmt and clippy --- src/ser.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ser.rs b/src/ser.rs index ecb995b..072212e 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -100,11 +100,10 @@ impl<'py> PythonizeMappingType<'py> for PyDict { /// both [`PythonizeTypes::Map`] and [`PythonizeTypes::NamedMap`] while only /// implementing [`PythonizeMappingType`]. pub struct PythonizeUnnamedMappingAdapter<'py, T: PythonizeMappingType<'py>> { - unnamed: T, + _unnamed: T, _marker: PhantomData<&'py ()>, } - impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> for PythonizeUnnamedMappingAdapter<'py, T> { From d5ab23fc39f1379d57c797279314995c53286630 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 8 Aug 2024 00:09:16 +0100 Subject: [PATCH 52/77] tidy up some lifetimes --- CHANGELOG.md | 23 ++++++++++----- src/de.rs | 81 ++++++++++++++++++++++++---------------------------- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c1c1f..7544af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,28 @@ ## Unreleased +### Packaging - Bump MSRV to 1.63 - Update to PyO3 0.22 + +### Added - Support `u128` / `i128` integers. -- Remove support for PyO3's `gil-refs` feature -- `pythonize()` now returns `Bound<'py, PyAny>` instead of `Py` -- `depythonize()` now take a `&Bound` and is no longer deprecated -- `depythonize_bound()` is now deprecated -- `Depythonizer` now contains a `&Bound` and so has an extra lifetime `'bound` -- `Depythonizer::from_object()` now takes a `&Bound` and is no longer deprecated -- Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` - Implement `PythonizeListType` for `PyTuple` - Support deserializing enums from any `PyMapping` instead of just `PyDict` - Support serializing struct-like types to named mappings using `PythonizeTypes::NamedMap` +### Changed +- `pythonize()` now returns `Bound<'py, PyAny>` instead of `Py` +- `depythonize()` now take `&'a Bound` and is no longer deprecated +- `depythonize_bound()` is now deprecated +- `Depythonizer::from_object()` now takes `&'a Bound` and is no longer deprecated +- `Depythonizer` now contains `&'a Bound` and so has an extra lifetime `'a` + +### Removed +- Remove support for PyO3's `gil-refs` feature + +### Fixed +- Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` + ## 0.21.1 - 2024-04-02 - Fix compile error when using PyO3 `abi3` feature targeting a minimum version below 3.10 diff --git a/src/de.rs b/src/de.rs index 986d0a6..d89fe33 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,43 +1,38 @@ use pyo3::{types::*, Bound}; -use serde::de::{self, IntoDeserializer}; +use serde::de::{self, DeserializeOwned, IntoDeserializer}; use serde::Deserialize; use crate::error::{PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` -pub fn depythonize<'py, 'obj, T>(obj: &'obj Bound<'py, PyAny>) -> Result +pub fn depythonize<'a, 'py, T>(obj: &'a Bound<'py, PyAny>) -> Result where - T: Deserialize<'obj>, + T: Deserialize<'a>, { - let mut depythonizer = Depythonizer::from_object(obj); - T::deserialize(&mut depythonizer) + T::deserialize(&mut Depythonizer::from_object(obj)) } /// Attempt to convert a Python object to an instance of `T` #[deprecated(since = "0.22.0", note = "use `depythonize` instead")] pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result where - T: for<'a> Deserialize<'a>, + T: DeserializeOwned, { - let mut depythonizer = Depythonizer::from_object(&obj); - T::deserialize(&mut depythonizer) + T::deserialize(&mut Depythonizer::from_object(&obj)) } /// A structure that deserializes Python objects into Rust values -pub struct Depythonizer<'py, 'bound> { - input: &'bound Bound<'py, PyAny>, +pub struct Depythonizer<'a, 'py> { + input: &'a Bound<'py, PyAny>, } -impl<'py, 'bound> Depythonizer<'py, 'bound> { +impl<'a, 'py> Depythonizer<'a, 'py> { /// Create a deserializer from a Python object - pub fn from_object<'input>(input: &'input Bound<'py, PyAny>) -> Depythonizer<'py, 'input> { + pub fn from_object(input: &'a Bound<'py, PyAny>) -> Self { Depythonizer { input } } - fn sequence_access( - &self, - expected_len: Option, - ) -> Result> { + fn sequence_access(&self, expected_len: Option) -> Result> { let seq = self.input.downcast::()?; let len = self.input.len()?; @@ -97,14 +92,14 @@ macro_rules! deserialize_type { }; } -impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, 'bound> { +impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { type Error = PythonizeError; fn deserialize_any(self, visitor: V) -> Result where V: de::Visitor<'de>, { - let obj = &self.input; + let obj = self.input; // First check for cases which are cheap to check due to pointer // comparison or bitflag checks @@ -307,8 +302,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' .downcast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; let value = m.get_item(&variant)?; - let mut de = Depythonizer::from_object(&value); - visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) + visitor.visit_enum(PyEnumAccess::new(&value, variant)) } else { Err(PythonizeError::invalid_enum_type()) } @@ -333,19 +327,19 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' } } -struct PySequenceAccess<'py, 'bound> { - seq: &'bound Bound<'py, PySequence>, +struct PySequenceAccess<'a, 'py> { + seq: &'a Bound<'py, PySequence>, index: usize, len: usize, } -impl<'py, 'bound> PySequenceAccess<'py, 'bound> { - fn new(seq: &'bound Bound<'py, PySequence>, len: usize) -> Self { +impl<'a, 'py> PySequenceAccess<'a, 'py> { + fn new(seq: &'a Bound<'py, PySequence>, len: usize) -> Self { Self { seq, index: 0, len } } } -impl<'de, 'py, 'bound> de::SeqAccess<'de> for PySequenceAccess<'py, 'bound> { +impl<'de> de::SeqAccess<'de> for PySequenceAccess<'_, '_> { type Error = PythonizeError; fn next_element_seed(&mut self, seed: T) -> Result> @@ -354,9 +348,9 @@ impl<'de, 'py, 'bound> de::SeqAccess<'de> for PySequenceAccess<'py, 'bound> { { if self.index < self.len { let item = self.seq.get_item(self.index)?; - let mut item_de = Depythonizer::from_object(&item); self.index += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } @@ -386,7 +380,7 @@ impl<'py> PyMappingAccess<'py> { } } -impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { +impl<'de> de::MapAccess<'de> for PyMappingAccess<'_> { type Error = PythonizeError; fn next_key_seed(&mut self, seed: K) -> Result> @@ -395,9 +389,9 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { { if self.key_idx < self.len { let item = self.keys.get_item(self.key_idx)?; - let mut item_de = Depythonizer::from_object(&item); self.key_idx += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } @@ -408,24 +402,26 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { V: de::DeserializeSeed<'de>, { let item = self.values.get_item(self.val_idx)?; - let mut item_de = Depythonizer::from_object(&item); self.val_idx += 1; - seed.deserialize(&mut item_de) + seed.deserialize(&mut Depythonizer::from_object(&item)) } } -struct PyEnumAccess<'a, 'py, 'bound> { - de: &'a mut Depythonizer<'py, 'bound>, +struct PyEnumAccess<'a, 'py> { + de: Depythonizer<'a, 'py>, variant: Bound<'py, PyString>, } -impl<'a, 'py, 'bound> PyEnumAccess<'a, 'py, 'bound> { - fn new(de: &'a mut Depythonizer<'py, 'bound>, variant: Bound<'py, PyString>) -> Self { - Self { de, variant } +impl<'a, 'py> PyEnumAccess<'a, 'py> { + fn new(obj: &'a Bound<'py, PyAny>, variant: Bound<'py, PyString>) -> Self { + Self { + de: Depythonizer::from_object(obj), + variant, + } } } -impl<'a, 'py, 'de, 'bound> de::EnumAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { +impl<'de> de::EnumAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; type Variant = Self; @@ -440,7 +436,7 @@ impl<'a, 'py, 'de, 'bound> de::EnumAccess<'de> for PyEnumAccess<'a, 'py, 'bound> } } -impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { +impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; fn unit_variant(self) -> Result<()> { @@ -451,7 +447,7 @@ impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bou where T: de::DeserializeSeed<'de>, { - seed.deserialize(self.de) + seed.deserialize(&mut { self.de }) } fn tuple_variant(self, len: usize, visitor: V) -> Result @@ -482,10 +478,7 @@ mod test { T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + let obj = py.eval_bound(code, None, None).unwrap(); let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); let actual_json: JsonValue = depythonize(&obj).unwrap(); From 12ab81765cbfc2fd0e0b46cf07d1d7469001439e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 19:11:36 +0100 Subject: [PATCH 53/77] add coverage for deprecated function --- src/de.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/de.rs b/src/de.rs index d89fe33..b1369c6 100644 --- a/src/de.rs +++ b/src/de.rs @@ -483,6 +483,10 @@ mod test { assert_eq!(&actual, expected); let actual_json: JsonValue = depythonize(&obj).unwrap(); assert_eq!(&actual_json, expected_json); + + #[allow(deprecated)] + let actual: T = depythonize_bound(obj.clone()).unwrap(); + assert_eq!(&actual, expected); }); } From 96f52ad989c6372c5e4b84d003895896f6ff6e8c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 21:22:53 +0100 Subject: [PATCH 54/77] add some more coverage --- src/de.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/de.rs b/src/de.rs index b1369c6..895e465 100644 --- a/src/de.rs +++ b/src/de.rs @@ -793,4 +793,39 @@ mod test { let _: i128 = depythonize(&i128::MIN.into_py(py).into_bound(py)).unwrap(); }); } + + #[test] + fn test_deserialize_bytes() { + Python::with_gil(|py| { + let obj = PyBytes::new_bound(py, "hello".as_bytes()); + let actual: Vec = depythonize(&obj).unwrap(); + assert_eq!(actual, b"hello"); + }) + } + + #[test] + fn test_char() { + let expected = 'a'; + let expected_json = json!("a"); + let code = "'a'"; + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_unknown_type() { + Python::with_gil(|py| { + let obj = py + .import_bound("decimal") + .unwrap() + .getattr("Decimal") + .unwrap() + .call0() + .unwrap(); + let err = depythonize::(&obj).unwrap_err(); + assert!(matches!( + *err.inner, + ErrorImpl::UnsupportedType(name) if name == "Decimal" + )); + }); + } } From cdc0f3b1490abdccc8b3c6d9c8706d37b1f5a59e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 21:46:59 +0100 Subject: [PATCH 55/77] serialize set and frozenset Co-authored-by: Lily Foote --- CHANGELOG.md | 1 + src/de.rs | 94 ++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7544af4..4283d66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ ### Fixed - Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` +- Fix deserializing `set` and `frozenset` into Rust sequences ## 0.21.1 - 2024-04-02 diff --git a/src/de.rs b/src/de.rs index b1369c6..51fd155 100644 --- a/src/de.rs +++ b/src/de.rs @@ -32,15 +32,28 @@ impl<'a, 'py> Depythonizer<'a, 'py> { Depythonizer { input } } - fn sequence_access(&self, expected_len: Option) -> Result> { - let seq = self.input.downcast::()?; + fn sequence_access(&self, expected_len: Option) -> Result> { + let seq = match self.input.downcast::() { + Ok(seq) => seq, + Err(e) => { + return if let Ok(set) = self.input.downcast::() { + Ok(SequenceAccess::Set(PySetAsSequence::from_set(&set))) + } else if let Ok(frozenset) = self.input.downcast::() { + Ok(SequenceAccess::Set(PySetAsSequence::from_frozenset( + &frozenset, + ))) + } else { + Err(e.into()) + } + } + }; let len = self.input.len()?; match expected_len { Some(expected) if expected != len => { Err(PythonizeError::incorrect_sequence_length(expected, len)) } - _ => Ok(PySequenceAccess::new(seq, len)), + _ => Ok(SequenceAccess::Sequence(PySequenceAccess::new(seq, len))), } } @@ -238,14 +251,20 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - visitor.visit_seq(self.sequence_access(None)?) + match self.sequence_access(None)? { + SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), + SequenceAccess::Set(set) => visitor.visit_seq(set), + } } fn deserialize_tuple(self, len: usize, visitor: V) -> Result where V: de::Visitor<'de>, { - visitor.visit_seq(self.sequence_access(Some(len))?) + match self.sequence_access(Some(len))? { + SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), + SequenceAccess::Set(set) => visitor.visit_seq(set), + } } fn deserialize_tuple_struct( @@ -257,7 +276,10 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - visitor.visit_seq(self.sequence_access(Some(len))?) + match self.sequence_access(Some(len))? { + SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), + SequenceAccess::Set(set) => visitor.visit_seq(set), + } } fn deserialize_map(self, visitor: V) -> Result @@ -327,6 +349,11 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { } } +enum SequenceAccess<'a, 'py> { + Sequence(PySequenceAccess<'a, 'py>), + Set(PySetAsSequence<'py>), +} + struct PySequenceAccess<'a, 'py> { seq: &'a Bound<'py, PySequence>, index: usize, @@ -357,6 +384,40 @@ impl<'de> de::SeqAccess<'de> for PySequenceAccess<'_, '_> { } } +struct PySetAsSequence<'py> { + iter: Bound<'py, PyIterator>, +} + +impl<'py> PySetAsSequence<'py> { + fn from_set(set: &Bound<'py, PySet>) -> Self { + Self { + iter: PyIterator::from_bound_object(&set).expect("set is always iterable"), + } + } + + fn from_frozenset(set: &Bound<'py, PyFrozenSet>) -> Self { + Self { + iter: PyIterator::from_bound_object(&set).expect("frozenset is always iterable"), + } + } +} + +impl<'de> de::SeqAccess<'de> for PySetAsSequence<'_> { + type Error = PythonizeError; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: de::DeserializeSeed<'de>, + { + match self.iter.next() { + Some(item) => seed + .deserialize(&mut Depythonizer::from_object(&item?)) + .map(Some), + None => Ok(None), + } + } +} + struct PyMappingAccess<'py> { keys: Bound<'py, PySequence>, values: Bound<'py, PySequence>, @@ -454,7 +515,10 @@ impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { where V: de::Visitor<'de>, { - visitor.visit_seq(self.de.sequence_access(Some(len))?) + match self.de.sequence_access(Some(len))? { + SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), + SequenceAccess::Set(set) => visitor.visit_seq(set), + } } fn struct_variant(self, _fields: &'static [&'static str], visitor: V) -> Result @@ -606,6 +670,22 @@ mod test { test_de(code, &expected, &expected_json); } + #[test] + fn test_tuple_from_pyset() { + let expected = ("foo".to_string(), 5); + let expected_json = json!(["foo", 5]); + let code = "{'foo', 5}"; + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_tuple_from_pyfrozenset() { + let expected = ("foo".to_string(), 5); + let expected_json = json!(["foo", 5]); + let code = "frozenset({'foo', 5})"; + test_de(code, &expected, &expected_json); + } + #[test] fn test_vec() { let expected = vec![3, 2, 1]; From 2666d63def4cd316d6a8f7a453bad16f34f57ad4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 22:12:35 +0100 Subject: [PATCH 56/77] only allow sets into homogeneous containers --- CHANGELOG.md | 2 +- src/de.rs | 91 ++++++++++++++++++++++++---------------------------- src/error.rs | 9 ++++++ 3 files changed, 52 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4283d66..30ff327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ ### Fixed - Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` -- Fix deserializing `set` and `frozenset` into Rust sequences +- Fix deserializing `set` and `frozenset` into Rust homogeneous containers ## 0.21.1 - 2024-04-02 diff --git a/src/de.rs b/src/de.rs index 51fd155..8b65825 100644 --- a/src/de.rs +++ b/src/de.rs @@ -2,7 +2,7 @@ use pyo3::{types::*, Bound}; use serde::de::{self, DeserializeOwned, IntoDeserializer}; use serde::Deserialize; -use crate::error::{PythonizeError, Result}; +use crate::error::{ErrorImpl, PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` pub fn depythonize<'a, 'py, T>(obj: &'a Bound<'py, PyAny>) -> Result @@ -32,28 +32,28 @@ impl<'a, 'py> Depythonizer<'a, 'py> { Depythonizer { input } } - fn sequence_access(&self, expected_len: Option) -> Result> { - let seq = match self.input.downcast::() { - Ok(seq) => seq, - Err(e) => { - return if let Ok(set) = self.input.downcast::() { - Ok(SequenceAccess::Set(PySetAsSequence::from_set(&set))) - } else if let Ok(frozenset) = self.input.downcast::() { - Ok(SequenceAccess::Set(PySetAsSequence::from_frozenset( - &frozenset, - ))) - } else { - Err(e.into()) - } - } - }; + fn sequence_access(&self, expected_len: Option) -> Result> { + let seq = self.input.downcast::()?; let len = self.input.len()?; match expected_len { Some(expected) if expected != len => { Err(PythonizeError::incorrect_sequence_length(expected, len)) } - _ => Ok(SequenceAccess::Sequence(PySequenceAccess::new(seq, len))), + _ => Ok(PySequenceAccess::new(seq, len)), + } + } + + fn set_access(&self) -> Result> { + match self.input.downcast::() { + Ok(set) => Ok(PySetAsSequence::from_set(&set)), + Err(e) => { + if let Ok(f) = self.input.downcast::() { + Ok(PySetAsSequence::from_frozenset(&f)) + } else { + Err(e.into()) + } + } } } @@ -135,10 +135,9 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { self.deserialize_bytes(visitor) } else if obj.is_instance_of::() { self.deserialize_f64(visitor) - } else if obj.is_instance_of::() - || obj.is_instance_of::() - || obj.downcast::().is_ok() - { + } else if obj.is_instance_of::() || obj.is_instance_of::() { + self.deserialize_seq(visitor) + } else if obj.downcast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.downcast::().is_ok() { self.deserialize_map(visitor) @@ -251,9 +250,17 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - match self.sequence_access(None)? { - SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), - SequenceAccess::Set(set) => visitor.visit_seq(set), + match self.sequence_access(None) { + Ok(seq) => visitor.visit_seq(seq), + Err(e) => { + // we allow sets to be deserialized as sequences, so try that + if matches!(*e.inner, ErrorImpl::UnexpectedType(_)) { + if let Ok(set) = self.set_access() { + return visitor.visit_seq(set); + } + } + Err(e) + } } } @@ -261,10 +268,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - match self.sequence_access(Some(len))? { - SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), - SequenceAccess::Set(set) => visitor.visit_seq(set), - } + visitor.visit_seq(self.sequence_access(Some(len))?) } fn deserialize_tuple_struct( @@ -276,10 +280,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - match self.sequence_access(Some(len))? { - SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), - SequenceAccess::Set(set) => visitor.visit_seq(set), - } + visitor.visit_seq(self.sequence_access(Some(len))?) } fn deserialize_map(self, visitor: V) -> Result @@ -349,11 +350,6 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { } } -enum SequenceAccess<'a, 'py> { - Sequence(PySequenceAccess<'a, 'py>), - Set(PySetAsSequence<'py>), -} - struct PySequenceAccess<'a, 'py> { seq: &'a Bound<'py, PySequence>, index: usize, @@ -515,10 +511,7 @@ impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { where V: de::Visitor<'de>, { - match self.de.sequence_access(Some(len))? { - SequenceAccess::Sequence(seq) => visitor.visit_seq(seq), - SequenceAccess::Set(set) => visitor.visit_seq(set), - } + visitor.visit_seq(self.de.sequence_access(Some(len))?) } fn struct_variant(self, _fields: &'static [&'static str], visitor: V) -> Result @@ -671,18 +664,18 @@ mod test { } #[test] - fn test_tuple_from_pyset() { - let expected = ("foo".to_string(), 5); - let expected_json = json!(["foo", 5]); - let code = "{'foo', 5}"; + fn test_vec_from_pyset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = "{'foo'}"; test_de(code, &expected, &expected_json); } #[test] - fn test_tuple_from_pyfrozenset() { - let expected = ("foo".to_string(), 5); - let expected_json = json!(["foo", 5]); - let code = "frozenset({'foo', 5})"; + fn test_vec_from_pyfrozenset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = "frozenset({'foo'})"; test_de(code, &expected, &expected_json); } diff --git a/src/error.rs b/src/error.rs index 4aee7ea..9aa5a87 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,6 +32,15 @@ impl PythonizeError { } } + pub(crate) fn unexpected_type(t: T) -> Self + where + T: ToString, + { + Self { + inner: Box::new(ErrorImpl::UnexpectedType(t.to_string())), + } + } + pub(crate) fn dict_key_not_string() -> Self { Self { inner: Box::new(ErrorImpl::DictKeyNotString), From 54d0932c9044c40d0f6c33affc18adfc71076fdd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 22:17:13 +0100 Subject: [PATCH 57/77] release: 0.22.0 --- Cargo.toml | 6 +++--- src/error.rs | 9 --------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 645ca32..415f5b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.21.1" +version = "0.22.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.63" @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.22.0", default-features = false } +pyo3 = { version = "0.22.2", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.22.2", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" diff --git a/src/error.rs b/src/error.rs index 9aa5a87..4aee7ea 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,15 +32,6 @@ impl PythonizeError { } } - pub(crate) fn unexpected_type(t: T) -> Self - where - T: ToString, - { - Self { - inner: Box::new(ErrorImpl::UnexpectedType(t.to_string())), - } - } - pub(crate) fn dict_key_not_string() -> Self { Self { inner: Box::new(ErrorImpl::DictKeyNotString), From eb0cad1c35816a439091fa43706bce3885d5b2dd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Aug 2024 22:22:55 +0100 Subject: [PATCH 58/77] add release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30ff327..7d584aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.22.0 - 2024-08-10 ### Packaging - Bump MSRV to 1.63 From 2b386e506e5270fbd0614b707f49956932922d0e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 11 Aug 2024 23:14:27 +0100 Subject: [PATCH 59/77] update `README.md` for newer APIs --- README.md | 30 +++++++++++------------------- src/lib.rs | 40 ++-------------------------------------- 2 files changed, 13 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 624e22d..441f29f 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,7 @@ that which is produced directly by `pythonize`. This crate converts Rust types which implement the [Serde] serialization traits into Python objects using the [PyO3] library. -Pythonize has two public APIs: `pythonize` and `depythonize_bound`. - - -

- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`, and pythonize is doing the same. +Pythonize has two main public APIs: `pythonize` and `depythonize`.
@@ -28,8 +21,8 @@ PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the ```rust use serde::{Serialize, Deserialize}; -use pyo3::Python; -use pythonize::{depythonize_bound, pythonize}; +use pyo3::prelude::*; +use pythonize::{depythonize, pythonize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct Sample { @@ -37,21 +30,20 @@ struct Sample { bar: Option } -let gil = Python::acquire_gil(); -let py = gil.python(); - let sample = Sample { foo: "Foo".to_string(), bar: None }; -// Rust -> Python -let obj = pythonize(py, &sample).unwrap(); +Python::with_gil(|py| { + // Rust -> Python + let obj = pythonize(py, &sample).unwrap(); -assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); + assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.repr().unwrap())); -// Python -> Rust -let new_sample: Sample = depythonize_bound(obj.into_bound(py)).unwrap(); + // Python -> Rust + let new_sample: Sample = depythonize(&obj).unwrap(); -assert_eq!(new_sample, sample); + assert_eq!(new_sample, sample); +}) ``` diff --git a/src/lib.rs b/src/lib.rs index 4ce1751..186fdf6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,41 +1,5 @@ -//! This crate converts Rust types which implement the [Serde] serialization -//! traits into Python objects using the [PyO3] library. -//! -//! Pythonize has two public APIs: `pythonize` and `depythonize`. -//! -//! [Serde]: https://github.com/serde-rs/serde -//! [PyO3]: https://github.com/PyO3/pyo3 -//! -//! # Examples -//! ``` -//! use serde::{Serialize, Deserialize}; -//! use pyo3::{types::PyAnyMethods, Python}; -//! use pythonize::{depythonize, pythonize}; -//! -//! #[derive(Debug, Serialize, Deserialize, PartialEq)] -//! struct Sample { -//! foo: String, -//! bar: Option -//! } -//! -//! Python::with_gil(|py| { -//! let sample = Sample { -//! foo: "Foo".to_string(), -//! bar: None -//! }; -//! -//! // Rust -> Python -//! let obj = pythonize(py, &sample).unwrap(); -//! -//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.repr().unwrap())); -//! -//! // Python -> Rust -//! let new_sample: Sample = depythonize(&obj).unwrap(); -//! -//! assert_eq!(new_sample, sample); -//! }); -//! -//! ``` +#![doc = include_str!("../README.md")] + mod de; mod error; mod ser; From b6742920fd6f39ae6fc6cb876c710d7d1d57070c Mon Sep 17 00:00:00 2001 From: Jonathan Coates Date: Thu, 21 Nov 2024 20:39:38 +0000 Subject: [PATCH 60/77] Update to PyO3 0.23 (#75) * Update to PyO3 0.23 - xyz_bound methods have been renamed to xyz - Use IntoPyObject for conversion to Python * Use c_str! instead of CStr literals * Add CHANGELOG entry * Cargo format --- CHANGELOG.md | 5 + Cargo.toml | 4 +- src/de.rs | 134 +++++++++++++-------------- src/error.rs | 8 ++ src/ser.rs | 94 +++++++++++-------- tests/test_custom_types.rs | 12 ++- tests/test_with_serde_path_to_err.rs | 16 ++-- 7 files changed, 150 insertions(+), 123 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d584aa..07455ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +### Packaging +- Update to PyO3 0.23 + ## 0.22.0 - 2024-08-10 ### Packaging diff --git a/Cargo.toml b/Cargo.toml index 415f5b9..1c25a12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.22.2", default-features = false } +pyo3 = { version = "0.23.1", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.22.2", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.23.1", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" diff --git a/src/de.rs b/src/de.rs index b2dcbdf..04e9740 100644 --- a/src/de.rs +++ b/src/de.rs @@ -14,7 +14,7 @@ where /// Attempt to convert a Python object to an instance of `T` #[deprecated(since = "0.22.0", note = "use `depythonize` instead")] -pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result +pub fn depythonize_bound(obj: Bound) -> Result where T: DeserializeOwned, { @@ -46,10 +46,10 @@ impl<'a, 'py> Depythonizer<'a, 'py> { fn set_access(&self) -> Result> { match self.input.downcast::() { - Ok(set) => Ok(PySetAsSequence::from_set(&set)), + Ok(set) => Ok(PySetAsSequence::from_set(set)), Err(e) => { if let Ok(f) = self.input.downcast::() { - Ok(PySetAsSequence::from_frozenset(&f)) + Ok(PySetAsSequence::from_frozenset(f)) } else { Err(e.into()) } @@ -387,13 +387,13 @@ struct PySetAsSequence<'py> { impl<'py> PySetAsSequence<'py> { fn from_set(set: &Bound<'py, PySet>) -> Self { Self { - iter: PyIterator::from_bound_object(&set).expect("set is always iterable"), + iter: PyIterator::from_object(set).expect("set is always iterable"), } } fn from_frozenset(set: &Bound<'py, PyFrozenSet>) -> Self { Self { - iter: PyIterator::from_bound_object(&set).expect("frozenset is always iterable"), + iter: PyIterator::from_object(set).expect("frozenset is always iterable"), } } } @@ -415,8 +415,8 @@ impl<'de> de::SeqAccess<'de> for PySetAsSequence<'_> { } struct PyMappingAccess<'py> { - keys: Bound<'py, PySequence>, - values: Bound<'py, PySequence>, + keys: Bound<'py, PyList>, + values: Bound<'py, PyList>, key_idx: usize, val_idx: usize, len: usize, @@ -524,18 +524,21 @@ impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { #[cfg(test)] mod test { + use std::ffi::CStr; + use super::*; use crate::error::ErrorImpl; use maplit::hashmap; - use pyo3::{IntoPy, Python}; + use pyo3::ffi::c_str; + use pyo3::{IntoPyObject, Python}; use serde_json::{json, Value as JsonValue}; - fn test_de(code: &str, expected: &T, expected_json: &JsonValue) + fn test_de(code: &CStr, expected: &T, expected_json: &JsonValue) where T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { Python::with_gil(|py| { - let obj = py.eval_bound(code, None, None).unwrap(); + let obj = py.eval(code, None, None).unwrap(); let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); let actual_json: JsonValue = depythonize(&obj).unwrap(); @@ -554,7 +557,7 @@ mod test { let expected = Empty; let expected_json = json!(null); - let code = "None"; + let code = c_str!("None"); test_de(code, &expected, &expected_json); } @@ -580,7 +583,7 @@ mod test { "baz": 45.23, "qux": true }); - let code = "{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"; + let code = c_str!("{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"); test_de(code, &expected, &expected_json); } @@ -592,13 +595,11 @@ mod test { bar: usize, } - let code = "{'foo': 'Foo'}"; + let code = c_str!("{'foo': 'Foo'}"); Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( *depythonize::(&obj).unwrap_err().inner, ErrorImpl::Message(msg) if msg == "missing field `bar`" @@ -613,7 +614,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "('cat', -10.05)"; + let code = c_str!("('cat', -10.05)"); test_de(code, &expected, &expected_json); } @@ -622,13 +623,11 @@ mod test { #[derive(Debug, Deserialize, PartialEq)] struct TupleStruct(String, f64); - let code = "('cat', -10.05, 'foo')"; + let code = c_str!("('cat', -10.05, 'foo')"); Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( *depythonize::(&obj).unwrap_err().inner, ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 @@ -643,7 +642,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "['cat', -10.05]"; + let code = c_str!("['cat', -10.05]"); test_de(code, &expected, &expected_json); } @@ -651,7 +650,7 @@ mod test { fn test_tuple() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "('foo', 5)"; + let code = c_str!("('foo', 5)"); test_de(code, &expected, &expected_json); } @@ -659,7 +658,7 @@ mod test { fn test_tuple_from_pylist() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "['foo', 5]"; + let code = c_str!("['foo', 5]"); test_de(code, &expected, &expected_json); } @@ -667,7 +666,7 @@ mod test { fn test_vec_from_pyset() { let expected = vec!["foo".to_string()]; let expected_json = json!(["foo"]); - let code = "{'foo'}"; + let code = c_str!("{'foo'}"); test_de(code, &expected, &expected_json); } @@ -675,7 +674,7 @@ mod test { fn test_vec_from_pyfrozenset() { let expected = vec!["foo".to_string()]; let expected_json = json!(["foo"]); - let code = "frozenset({'foo'})"; + let code = c_str!("frozenset({'foo'})"); test_de(code, &expected, &expected_json); } @@ -683,7 +682,7 @@ mod test { fn test_vec() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "[3, 2, 1]"; + let code = c_str!("[3, 2, 1]"); test_de(code, &expected, &expected_json); } @@ -691,7 +690,7 @@ mod test { fn test_vec_from_tuple() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "(3, 2, 1)"; + let code = c_str!("(3, 2, 1)"); test_de(code, &expected, &expected_json); } @@ -699,7 +698,7 @@ mod test { fn test_hashmap() { let expected = hashmap! {"foo".to_string() => 4}; let expected_json = json!({"foo": 4 }); - let code = "{'foo': 4}"; + let code = c_str!("{'foo': 4}"); test_de(code, &expected, &expected_json); } @@ -712,7 +711,7 @@ mod test { let expected = Foo::Variant; let expected_json = json!("Variant"); - let code = "'Variant'"; + let code = c_str!("'Variant'"); test_de(code, &expected, &expected_json); } @@ -725,7 +724,7 @@ mod test { let expected = Foo::Tuple(12, "cat".to_string()); let expected_json = json!({"Tuple": [12, "cat"]}); - let code = "{'Tuple': [12, 'cat']}"; + let code = c_str!("{'Tuple': [12, 'cat']}"); test_de(code, &expected, &expected_json); } @@ -738,7 +737,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!({"NewType": "cat" }); - let code = "{'NewType': 'cat'}"; + let code = c_str!("{'NewType': 'cat'}"); test_de(code, &expected, &expected_json); } @@ -754,7 +753,7 @@ mod test { bar: 25, }; let expected_json = json!({"Struct": {"foo": "cat", "bar": 25 }}); - let code = "{'Struct': {'foo': 'cat', 'bar': 25}}"; + let code = c_str!("{'Struct': {'foo': 'cat', 'bar': 25}}"); test_de(code, &expected, &expected_json); } #[test] @@ -767,7 +766,7 @@ mod test { let expected = Foo::Tuple(12.0, 'c'); let expected_json = json!([12.0, 'c']); - let code = "[12.0, 'c']"; + let code = c_str!("[12.0, 'c']"); test_de(code, &expected, &expected_json); } @@ -781,7 +780,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!("cat"); - let code = "'cat'"; + let code = c_str!("'cat'"); test_de(code, &expected, &expected_json); } @@ -798,7 +797,7 @@ mod test { bar: [2, 5, 3, 1], }; let expected_json = json!({"foo": ["a", "b", "c"], "bar": [2, 5, 3, 1]}); - let code = "{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"; + let code = c_str!("{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"); test_de(code, &expected, &expected_json); } @@ -831,7 +830,8 @@ mod test { }; let expected_json = json!({"name": "SomeFoo", "bar": { "value": 13, "variant": { "Tuple": [-1.5, 8]}}}); - let code = "{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"; + let code = + c_str!("{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"); test_de(code, &expected, &expected_json); } @@ -839,38 +839,38 @@ mod test { fn test_int_limits() { Python::with_gil(|py| { // serde_json::Value supports u64 and i64 as maxiumum sizes - let _: serde_json::Value = depythonize(&u8::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u8::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i8::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i8::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u16::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u16::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i16::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i16::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u32::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u32::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i32::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i32::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i64::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: u128 = depythonize(&u128::MAX.into_py(py).into_bound(py)).unwrap(); - let _: i128 = depythonize(&u128::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: i128 = depythonize(&i128::MAX.into_py(py).into_bound(py)).unwrap(); - let _: i128 = depythonize(&i128::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u8::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u16::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u32::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u64::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: u128 = depythonize(&u128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&u128::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: i128 = depythonize(&i128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&i128::MIN.into_pyobject(py).unwrap()).unwrap(); }); } #[test] fn test_deserialize_bytes() { Python::with_gil(|py| { - let obj = PyBytes::new_bound(py, "hello".as_bytes()); + let obj = PyBytes::new(py, "hello".as_bytes()); let actual: Vec = depythonize(&obj).unwrap(); assert_eq!(actual, b"hello"); }) @@ -880,7 +880,7 @@ mod test { fn test_char() { let expected = 'a'; let expected_json = json!("a"); - let code = "'a'"; + let code = c_str!("'a'"); test_de(code, &expected, &expected_json); } @@ -888,7 +888,7 @@ mod test { fn test_unknown_type() { Python::with_gil(|py| { let obj = py - .import_bound("decimal") + .import("decimal") .unwrap() .getattr("Decimal") .unwrap() diff --git a/src/error.rs b/src/error.rs index 4aee7ea..1bcc556 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ use pyo3::PyErr; use pyo3::{exceptions::*, DowncastError, DowncastIntoError}; use serde::{de, ser}; +use std::convert::Infallible; use std::error; use std::fmt::{self, Debug, Display}; use std::result; @@ -136,6 +137,13 @@ impl de::Error for PythonizeError { } } +/// Convert an exception raised in Python to a `PythonizeError` +impl From for PythonizeError { + fn from(other: Infallible) -> Self { + match other {} + } +} + /// Convert an exception raised in Python to a `PythonizeError` impl From for PythonizeError { fn from(other: PyErr) -> Self { diff --git a/src/ser.rs b/src/ser.rs index 072212e..efc7e29 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,10 +1,10 @@ use std::marker::PhantomData; use pyo3::types::{ - PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PySequence, PyString, PyTuple, + PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PySequence, PyString, PyTuple, PyTupleMethods, }; -use pyo3::{Bound, IntoPy, PyAny, PyResult, Python, ToPyObject}; +use pyo3::{Bound, BoundObject, IntoPyObject, PyAny, PyResult, Python}; use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; @@ -52,12 +52,12 @@ pub trait PythonizeNamedMappingType<'py> { /// Trait for types which can represent a Python sequence pub trait PythonizeListType: Sized { /// Constructor - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator; } @@ -76,7 +76,7 @@ impl<'py> PythonizeMappingType<'py> for PyDict { type Builder = Bound<'py, Self>; fn builder(py: Python<'py>, _len: Option) -> PyResult { - Ok(Self::new_bound(py)) + Ok(Self::new(py)) } fn push_item( @@ -127,31 +127,28 @@ impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> } impl PythonizeListType for PyList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { - Ok(PyList::new_bound(py, elements) - .into_any() - .downcast_into() - .unwrap()) + Ok(PyList::new(py, elements)?.into_sequence()) } } impl PythonizeListType for PyTuple { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { - Ok(PyTuple::new_bound(py, elements).into_sequence()) + Ok(PyTuple::new(py, elements)?.into_sequence()) } } @@ -245,6 +242,20 @@ pub struct PythonMapSerializer<'py, P: PythonizeTypes<'py>> { _types: PhantomData

, } +impl<'py, P: PythonizeTypes<'py>> Pythonizer<'py, P> { + /// The default implementation for serialisation functions. + #[inline] + fn serialise_default(self, v: T) -> Result> + where + T: IntoPyObject<'py>, + >::Error: Into, + { + v.into_pyobject(self.py) + .map(|x| x.into_any().into_bound()) + .map_err(Into::into) + } +} + impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -257,47 +268,47 @@ impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { type SerializeStructVariant = PythonStructVariantSerializer<'py, P>; fn serialize_bool(self, v: bool) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i8(self, v: i8) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i16(self, v: i16) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i32(self, v: i32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i64(self, v: i64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u8(self, v: u8) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u16(self, v: u16) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u32(self, v: u32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u64(self, v: u64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_f32(self, v: f32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_f64(self, v: f64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_char(self, v: char) -> Result> { @@ -305,11 +316,11 @@ impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { } fn serialize_str(self, v: &str) -> Result> { - Ok(PyString::new_bound(self.py, v).into_any()) + Ok(PyString::new(self.py, v).into_any()) } fn serialize_bytes(self, v: &[u8]) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_none(self) -> Result> { @@ -364,7 +375,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { let mut m = P::NamedMap::builder(self.py, 1, name)?; P::NamedMap::push_field( &mut m, - PyString::new_bound(self.py, variant), + PyString::new(self.py, variant), value.serialize(self)?, )?; Ok(P::NamedMap::finish(m)?.into_any()) @@ -467,7 +478,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeSeq for PythonCollectionSerializ fn end(self) -> Result> { let instance = P::List::create_sequence(self.py, self.items)?; - Ok(instance.to_object(self.py).into_bound(self.py)) + Ok(instance.into_pyobject(self.py)?.into_any()) } } @@ -483,7 +494,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeTuple for PythonCollectionSerial } fn end(self) -> Result> { - Ok(PyTuple::new_bound(self.py, self.items).into_any()) + Ok(PyTuple::new(self.py, self.items)?.into_any()) } } @@ -520,7 +531,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleVariant let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; P::NamedMap::push_field( &mut m, - PyString::new_bound(self.inner.py, self.variant), + PyString::new(self.inner.py, self.variant), ser::SerializeTuple::end(self.inner)?, )?; Ok(P::NamedMap::finish(m)?.into_any()) @@ -568,7 +579,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeStruct for PythonStructDictSeria { P::NamedMap::push_field( &mut self.builder, - PyString::new_bound(self.py, key), + PyString::new(self.py, key), pythonize_custom::(self.py, value)?, )?; Ok(()) @@ -591,7 +602,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeStructVariant { P::NamedMap::push_field( &mut self.inner.builder, - PyString::new_bound(self.inner.py, key), + PyString::new(self.inner.py, key), pythonize_custom::(self.inner.py, value)?, )?; Ok(()) @@ -602,7 +613,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeStructVariant let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; P::NamedMap::push_field( &mut m, - PyString::new_bound(self.inner.py, self.variant), + PyString::new(self.inner.py, self.variant), v.into_any(), )?; Ok(P::NamedMap::finish(m)?.into_any()) @@ -613,6 +624,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeStructVariant mod test { use super::pythonize; use maplit::hashmap; + use pyo3::ffi::c_str; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyBytes, PyDict}; @@ -625,11 +637,11 @@ mod test { Python::with_gil(|py| -> PyResult<()> { let obj = pythonize(py, &src)?; - let locals = PyDict::new_bound(py); + let locals = PyDict::new(py); locals.set_item("obj", obj)?; - py.run_bound( - "import json; result = json.dumps(obj, separators=(',', ':'))", + py.run( + c_str!("import json; result = json.dumps(obj, separators=(',', ':'))"), None, Some(&locals), )?; @@ -838,7 +850,7 @@ mod test { Python::with_gil(|py| { assert!(pythonize(py, serde_bytes::Bytes::new(b"foo")) .expect("bytes will always serialize successfully") - .eq(&PyBytes::new_bound(py, b"foo")) + .eq(&PyBytes::new(py, b"foo")) .expect("bytes will always compare successfully")); }); } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 5f0084b..1f5dfd4 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -4,6 +4,7 @@ use pyo3::{ exceptions::{PyIndexError, PyKeyError}, prelude::*, types::{PyDict, PyMapping, PySequence, PyTuple}, + BoundObject, }; use pythonize::{ depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, @@ -32,12 +33,12 @@ impl CustomList { } impl PythonizeListType for CustomList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { let sequence = Bound::new( @@ -45,8 +46,9 @@ impl PythonizeListType for CustomList { CustomList { items: elements .into_iter() - .map(|item| item.to_object(py)) - .collect(), + .map(|item| item.into_pyobject(py).map(|x| x.into_any().unbind())) + .collect::, T::Error>>() + .map_err(Into::into)?, }, )? .into_any(); diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 1321c2b..5f2b970 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -41,14 +41,14 @@ impl Serialize for CannotSerialize { #[test] fn test_de_valid() { Python::with_gil(|py| { - let pyroot = PyDict::new_bound(py); + let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new_bound(py); - let nested_0 = PyDict::new_bound(py); + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new_bound(py); + let nested_1 = PyDict::new(py); nested_1.set_item("nested_key", "nested_value_1").unwrap(); nested.set_item("nested_1", nested_1).unwrap(); @@ -83,14 +83,14 @@ fn test_de_valid() { #[test] fn test_de_invalid() { Python::with_gil(|py| { - let pyroot = PyDict::new_bound(py); + let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new_bound(py); - let nested_0 = PyDict::new_bound(py); + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new_bound(py); + let nested_1 = PyDict::new(py); nested_1.set_item("nested_key", 1).unwrap(); nested.set_item("nested_1", nested_1).unwrap(); From bc56c21c3d5dd98759e17ba08bdad059f18c9c64 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Nov 2024 12:37:30 +0000 Subject: [PATCH 61/77] ci: add Python 3.13 and 3.13t testing (#76) * ci: add Python 3.13 and 3.13t testing * skip freethreaded + abi3 combination --- .github/workflows/ci.yml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62b3e2d..d323b30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,21 +34,17 @@ jobs: name: python${{ matrix.python-version }} ${{ matrix.os }} rust-${{ matrix.rust}} runs-on: ${{ matrix.os }} strategy: - fail-fast: false # If one platform fails, allow the rest to keep testing. + fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: python-architecture: ["x64"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ - "macos-13", - "ubuntu-latest", - "windows-latest", - ] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] + os: ["macos-13", "ubuntu-latest", "windows-latest"] rust: [stable] include: - - python-version: "3.12" + - python-version: "3.13" os: "ubuntu-latest" rust: "1.63" - - python-version: "3.12" + - python-version: "3.13" python-architecture: "arm64" os: "macos-latest" rust: "stable" @@ -57,7 +53,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: Quansight-Labs/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.python-python-architecture }} @@ -73,7 +69,9 @@ jobs: - name: Test run: cargo test --verbose - - name: Test (abi3) + # https://github.com/PyO3/pyo3/issues/4709 - can't use abi3 w. freethreaded build + - if: ${{ !endsWith(matrix.python-version, 't') }} + name: Test (abi3) run: cargo test --verbose --features pyo3/abi3-py37 env: From 4491cdb46f35ab3fc1b419beef92171fbef510da Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Nov 2024 16:54:08 +0000 Subject: [PATCH 62/77] release: 0.23 (#77) --- CHANGELOG.md | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07455ab..c4a1633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.23.0 - 2024-11-22 ### Packaging - Update to PyO3 0.23 diff --git a/Cargo.toml b/Cargo.toml index 1c25a12..f696dea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.22.0" +version = "0.23.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.63" From 637c09f39d82aba7bf8e780cfd59ed874519c819 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 20 Mar 2025 20:23:42 +0000 Subject: [PATCH 63/77] ci: resolve MSRV from Cargo.toml (#82) * ci: resolve MSRV from Cargo.toml * downgrade dependencies for MSRV compatibility --- .github/workflows/ci.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d323b30..a09182b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,19 @@ env: CARGO_TERM_COLOR: always jobs: + resolve: + runs-on: ubuntu-latest + outputs: + MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: resolve MSRV + id: resolve-msrv + run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT + fmt: runs-on: ubuntu-latest steps: @@ -30,7 +43,7 @@ jobs: - run: cargo clippy --all build: - needs: [fmt] # don't wait for clippy as fails rarely and takes longer + needs: [resolve, fmt] # don't wait for clippy as fails rarely and takes longer name: python${{ matrix.python-version }} ${{ matrix.os }} rust-${{ matrix.rust}} runs-on: ${{ matrix.os }} strategy: @@ -43,7 +56,7 @@ jobs: include: - python-version: "3.13" os: "ubuntu-latest" - rust: "1.63" + rust: ${{ needs.resolve.outputs.MSRV }} - python-version: "3.13" python-architecture: "arm64" os: "macos-latest" @@ -66,6 +79,12 @@ jobs: - uses: Swatinem/rust-cache@v2 continue-on-error: true + - if: ${{ matrix.rust == needs.resolve.outputs.MSRV }} + name: Set dependencies on MSRV + run: cargo +stable update + env: + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback + - name: Test run: cargo test --verbose From f2947c9eff9bb532c2ce6361ec96e02b895ca08a Mon Sep 17 00:00:00 2001 From: Gernot Bauer Date: Thu, 20 Mar 2025 21:29:49 +0100 Subject: [PATCH 64/77] Update PyO3 version to 0.24 (#81) Co-authored-by: David Hewitt --- CHANGELOG.md | 5 +++++ Cargo.toml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a1633..58ab193 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Unreleased + +### Packaging +- Update to PyO3 0.24 + ## 0.23.0 - 2024-11-22 ### Packaging diff --git a/Cargo.toml b/Cargo.toml index f696dea..57de5e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.23.0" +version = "0.24.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.63" @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.23.1", default-features = false } +pyo3 = { version = "0.24", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.23.1", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.24", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" From b51c42eab0045ae0e00b07b4563bb6dca7c2f7b6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 20 Mar 2025 20:47:00 +0000 Subject: [PATCH 65/77] remove `depythonize_bound` (#83) * remove `depythonize_bound` * cleanup --- CHANGELOG.md | 3 +++ src/de.rs | 16 ++-------------- src/lib.rs | 2 -- src/ser.rs | 6 +++--- tests/test_custom_types.rs | 2 +- 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ab193..8371995 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ### Packaging - Update to PyO3 0.24 +## Removed +- Remove deprecated `depythonize_bound()` + ## 0.23.0 - 2024-11-22 ### Packaging diff --git a/src/de.rs b/src/de.rs index 04e9740..49c0889 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,5 +1,5 @@ use pyo3::{types::*, Bound}; -use serde::de::{self, DeserializeOwned, IntoDeserializer}; +use serde::de::{self, IntoDeserializer}; use serde::Deserialize; use crate::error::{ErrorImpl, PythonizeError, Result}; @@ -12,15 +12,6 @@ where T::deserialize(&mut Depythonizer::from_object(obj)) } -/// Attempt to convert a Python object to an instance of `T` -#[deprecated(since = "0.22.0", note = "use `depythonize` instead")] -pub fn depythonize_bound(obj: Bound) -> Result -where - T: DeserializeOwned, -{ - T::deserialize(&mut Depythonizer::from_object(&obj)) -} - /// A structure that deserializes Python objects into Rust values pub struct Depythonizer<'a, 'py> { input: &'a Bound<'py, PyAny>, @@ -541,12 +532,9 @@ mod test { let obj = py.eval(code, None, None).unwrap(); let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); + let actual_json: JsonValue = depythonize(&obj).unwrap(); assert_eq!(&actual_json, expected_json); - - #[allow(deprecated)] - let actual: T = depythonize_bound(obj.clone()).unwrap(); - assert_eq!(&actual, expected); }); } diff --git a/src/lib.rs b/src/lib.rs index 186fdf6..e625b6f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,8 +4,6 @@ mod de; mod error; mod ser; -#[allow(deprecated)] -pub use crate::de::depythonize_bound; pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ diff --git a/src/ser.rs b/src/ser.rs index efc7e29..513a8e2 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -55,7 +55,7 @@ pub trait PythonizeListType: Sized { fn create_sequence<'py, T, U>( py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where T: IntoPyObject<'py>, U: ExactSizeIterator; @@ -130,7 +130,7 @@ impl PythonizeListType for PyList { fn create_sequence<'py, T, U>( py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where T: IntoPyObject<'py>, U: ExactSizeIterator, @@ -143,7 +143,7 @@ impl PythonizeListType for PyTuple { fn create_sequence<'py, T, U>( py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where T: IntoPyObject<'py>, U: ExactSizeIterator, diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 1f5dfd4..d311c14 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -36,7 +36,7 @@ impl PythonizeListType for CustomList { fn create_sequence<'py, T, U>( py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where T: IntoPyObject<'py>, U: ExactSizeIterator, From 5ed610664ec1821051958d713654d59d42fce5b9 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 26 Mar 2025 15:19:25 -0600 Subject: [PATCH 66/77] replace quansight-labs/setup-python with actions/setup-python (#84) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a09182b..6c08531 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: Quansight-Labs/setup-python@v5 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.python-python-architecture }} From 3c7d7cef98191f7d9fcc675f72f18b7f63a3ceaa Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 26 Mar 2025 21:20:17 +0000 Subject: [PATCH 67/77] release: 0.24.0 (#85) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8371995..16b1080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Unreleased +## 0.24.0 - 2025-03-26 ### Packaging - Update to PyO3 0.24 From 49ee947d5f341607f63504a9a9549f67edea81e0 Mon Sep 17 00:00:00 2001 From: Dylan DPC <99973273+Dylan-DPC@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:30:41 +0530 Subject: [PATCH 68/77] Update Cargo.toml (#87) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 57de5e6..8903051 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ pyo3 = { version = "0.24", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.24", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.24.1", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" From 6f51e936d926e62b229f0ee850a699acaf66ce65 Mon Sep 17 00:00:00 2001 From: jesse Date: Fri, 23 May 2025 04:34:01 -0700 Subject: [PATCH 69/77] update pyo3 version to 0.25.x (#88) --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8903051..b9d4df2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.24.0" +version = "0.25.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.63" @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.24", default-features = false } +pyo3 = { version = "0.25", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.24.1", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.25", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" From 096f83e7b7ec436524c6561d3a99be5159e9bee1 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 May 2025 12:35:38 +0100 Subject: [PATCH 70/77] add ci to release crate (#89) --- .github/release.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/release.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..5f21b46 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,21 @@ +name: Release Rust Crate + +on: + push: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + environment: release + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Publish to crates.io + run: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} From 435870cd014456dc13a384434f6f43838152c459 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 May 2025 12:38:38 +0100 Subject: [PATCH 71/77] fixup workflow location (#90) --- .github/{ => workflows}/release.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/release.yml (100%) diff --git a/.github/release.yml b/.github/workflows/release.yml similarity index 100% rename from .github/release.yml rename to .github/workflows/release.yml From ddfe9f111e8b2f07ec0e5ee5f211074b2880cd73 Mon Sep 17 00:00:00 2001 From: jesse Date: Sat, 14 Jun 2025 03:02:14 -0700 Subject: [PATCH 72/77] fix-typos (#92) --- src/de.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/de.rs b/src/de.rs index 49c0889..2860e48 100644 --- a/src/de.rs +++ b/src/de.rs @@ -121,7 +121,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { self.deserialize_str(visitor) } // Continue with cases which are slower to check because they go - // throuh `isinstance` machinery + // through `isinstance` machinery else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_bytes(visitor) } else if obj.is_instance_of::() { @@ -826,7 +826,7 @@ mod test { #[test] fn test_int_limits() { Python::with_gil(|py| { - // serde_json::Value supports u64 and i64 as maxiumum sizes + // serde_json::Value supports u64 and i64 as maximum sizes let _: serde_json::Value = depythonize(&u8::MAX.into_pyobject(py).unwrap()).unwrap(); let _: serde_json::Value = depythonize(&u8::MIN.into_pyobject(py).unwrap()).unwrap(); let _: serde_json::Value = depythonize(&i8::MAX.into_pyobject(py).unwrap()).unwrap(); From e64436cd21504e3ebd6bb25da0ea36ca42c1e8a3 Mon Sep 17 00:00:00 2001 From: jesse Date: Sat, 30 Aug 2025 01:53:40 -0700 Subject: [PATCH 73/77] did I `GAT` this right? (#91) --- Cargo.toml | 2 +- src/ser.rs | 114 +++++++++++++-------------- tests/test_custom_types.rs | 38 +++++---- tests/test_with_serde_path_to_err.rs | 4 +- 4 files changed, 80 insertions(+), 78 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b9d4df2..d636384 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "pythonize" version = "0.25.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" -rust-version = "1.63" +rust-version = "1.65" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" diff --git a/src/ser.rs b/src/ser.rs index 513a8e2..dce22e3 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -9,44 +9,46 @@ use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; -// TODO: move 'py lifetime into builder once GATs are available in MSRV /// Trait for types which can represent a Python mapping -pub trait PythonizeMappingType<'py> { +pub trait PythonizeMappingType { /// Builder type for Python mappings - type Builder; + type Builder<'py>: 'py; /// Create a builder for a Python mapping - fn builder(py: Python<'py>, len: Option) -> PyResult; + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult>; /// Adds the key-value item to the mapping being built - fn push_item( - builder: &mut Self::Builder, + fn push_item<'py>( + builder: &mut Self::Builder<'py>, key: Bound<'py, PyAny>, value: Bound<'py, PyAny>, ) -> PyResult<()>; /// Build the Python mapping - fn finish(builder: Self::Builder) -> PyResult>; + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; } -// TODO: move 'py lifetime into builder once GATs are available in MSRV /// Trait for types which can represent a Python mapping and have a name -pub trait PythonizeNamedMappingType<'py> { +pub trait PythonizeNamedMappingType { /// Builder type for Python mappings with a name - type Builder; + type Builder<'py>: 'py; /// Create a builder for a Python mapping with a name - fn builder(py: Python<'py>, len: usize, name: &'static str) -> PyResult; + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult>; /// Adds the field to the named mapping being built - fn push_field( - builder: &mut Self::Builder, + fn push_field<'py>( + builder: &mut Self::Builder<'py>, name: Bound<'py, PyString>, value: Bound<'py, PyAny>, ) -> PyResult<()>; /// Build the Python mapping - fn finish(builder: Self::Builder) -> PyResult>; + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; } /// Trait for types which can represent a Python sequence @@ -61,33 +63,32 @@ pub trait PythonizeListType: Sized { U: ExactSizeIterator; } -// TODO: remove 'py lifetime once GATs are available in MSRV /// Custom types for serialization -pub trait PythonizeTypes<'py> { +pub trait PythonizeTypes { /// Python map type (should be representable as python mapping) - type Map: PythonizeMappingType<'py>; + type Map: PythonizeMappingType; /// Python (struct-like) named map type (should be representable as python mapping) - type NamedMap: PythonizeNamedMappingType<'py>; + type NamedMap: PythonizeNamedMappingType; /// Python sequence type (should be representable as python sequence) type List: PythonizeListType; } -impl<'py> PythonizeMappingType<'py> for PyDict { - type Builder = Bound<'py, Self>; +impl PythonizeMappingType for PyDict { + type Builder<'py> = Bound<'py, Self>; - fn builder(py: Python<'py>, _len: Option) -> PyResult { + fn builder<'py>(py: Python<'py>, _len: Option) -> PyResult> { Ok(Self::new(py)) } - fn push_item( - builder: &mut Self::Builder, + fn push_item<'py>( + builder: &mut Self::Builder<'py>, key: Bound<'py, PyAny>, value: Bound<'py, PyAny>, ) -> PyResult<()> { builder.set_item(key, value) } - fn finish(builder: Self::Builder) -> PyResult> { + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { Ok(builder.into_mapping()) } } @@ -99,30 +100,31 @@ impl<'py> PythonizeMappingType<'py> for PyDict { /// This adapter is commonly applied to use the same unnamed mapping type for /// both [`PythonizeTypes::Map`] and [`PythonizeTypes::NamedMap`] while only /// implementing [`PythonizeMappingType`]. -pub struct PythonizeUnnamedMappingAdapter<'py, T: PythonizeMappingType<'py>> { +pub struct PythonizeUnnamedMappingAdapter { _unnamed: T, - _marker: PhantomData<&'py ()>, } -impl<'py, T: PythonizeMappingType<'py>> PythonizeNamedMappingType<'py> - for PythonizeUnnamedMappingAdapter<'py, T> -{ - type Builder = >::Builder; +impl PythonizeNamedMappingType for PythonizeUnnamedMappingAdapter { + type Builder<'py> = T::Builder<'py>; - fn builder(py: Python<'py>, len: usize, _name: &'static str) -> PyResult { - ::builder(py, Some(len)) + fn builder<'py>( + py: Python<'py>, + len: usize, + _name: &'static str, + ) -> PyResult> { + T::builder(py, Some(len)) } - fn push_field( - builder: &mut Self::Builder, + fn push_field<'py>( + builder: &mut Self::Builder<'py>, name: Bound<'py, PyString>, value: Bound<'py, PyAny>, ) -> PyResult<()> { - ::push_item(builder, name.into_any(), value) + T::push_item(builder, name.into_any(), value) } - fn finish(builder: Self::Builder) -> PyResult> { - ::finish(builder) + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + T::finish(builder) } } @@ -154,9 +156,9 @@ impl PythonizeListType for PyTuple { pub struct PythonizeDefault; -impl<'py> PythonizeTypes<'py> for PythonizeDefault { +impl PythonizeTypes for PythonizeDefault { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyList; } @@ -173,7 +175,7 @@ where pub fn pythonize_custom<'py, P, T>(py: Python<'py>, value: &T) -> Result> where T: ?Sized + Serialize, - P: PythonizeTypes<'py>, + P: PythonizeTypes, { value.serialize(Pythonizer::custom::

(py)) } @@ -221,28 +223,28 @@ pub struct PythonTupleVariantSerializer<'py, P> { } #[doc(hidden)] -pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes<'py>> { +pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { name: &'static str, variant: &'static str, inner: PythonStructDictSerializer<'py, P>, } #[doc(hidden)] -pub struct PythonStructDictSerializer<'py, P: PythonizeTypes<'py>> { +pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - builder: >::Builder, + builder: ::Builder<'py>, _types: PhantomData

, } #[doc(hidden)] -pub struct PythonMapSerializer<'py, P: PythonizeTypes<'py>> { +pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - builder: >::Builder, + builder: ::Builder<'py>, key: Option>, _types: PhantomData

, } -impl<'py, P: PythonizeTypes<'py>> Pythonizer<'py, P> { +impl<'py, P: PythonizeTypes> Pythonizer<'py, P> { /// The default implementation for serialisation functions. #[inline] fn serialise_default(self, v: T) -> Result> @@ -256,7 +258,7 @@ impl<'py, P: PythonizeTypes<'py>> Pythonizer<'py, P> { } } -impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { +impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; type SerializeSeq = PythonCollectionSerializer<'py, P>; @@ -464,7 +466,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::Serializer for Pythonizer<'py, P> { } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeSeq for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -482,7 +484,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeSeq for PythonCollectionSerializ } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeTuple for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -498,7 +500,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeTuple for PythonCollectionSerial } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleStruct for PythonCollectionSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeTupleStruct for PythonCollectionSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -514,9 +516,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleStruct for PythonCollection } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleVariant - for PythonTupleVariantSerializer<'py, P> -{ +impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -538,7 +538,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeTupleVariant } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeMap for PythonMapSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -569,7 +569,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeMap for PythonMapSerializer<'py, } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -590,9 +590,7 @@ impl<'py, P: PythonizeTypes<'py>> ser::SerializeStruct for PythonStructDictSeria } } -impl<'py, P: PythonizeTypes<'py>> ser::SerializeStructVariant - for PythonStructVariantSerializer<'py, P> -{ +impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariantSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index d311c14..32c5768 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -58,9 +58,9 @@ impl PythonizeListType for CustomList { } struct PythonizeCustomList; -impl<'py> PythonizeTypes<'py> for PythonizeCustomList { +impl<'py> PythonizeTypes for PythonizeCustomList { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = CustomList; } @@ -107,10 +107,10 @@ impl CustomDict { } } -impl<'py> PythonizeMappingType<'py> for CustomDict { - type Builder = Bound<'py, CustomDict>; +impl PythonizeMappingType for CustomDict { + type Builder<'py> = Bound<'py, CustomDict>; - fn builder(py: Python<'py>, len: Option) -> PyResult { + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult> { Bound::new( py, CustomDict { @@ -119,23 +119,23 @@ impl<'py> PythonizeMappingType<'py> for CustomDict { ) } - fn push_item( - builder: &mut Self::Builder, + fn push_item<'py>( + builder: &mut Self::Builder<'py>, key: Bound<'py, PyAny>, value: Bound<'py, PyAny>, ) -> PyResult<()> { unsafe { builder.downcast_unchecked::() }.set_item(key, value) } - fn finish(builder: Self::Builder) -> PyResult> { + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { Ok(unsafe { builder.into_any().downcast_into_unchecked() }) } } struct PythonizeCustomDict; -impl<'py> PythonizeTypes<'py> for PythonizeCustomDict { +impl<'py> PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; - type NamedMap = PythonizeUnnamedMappingAdapter<'py, CustomDict>; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyTuple; } @@ -215,10 +215,14 @@ impl NamedCustomDict { } } -impl<'py> PythonizeNamedMappingType<'py> for NamedCustomDict { - type Builder = Bound<'py, NamedCustomDict>; +impl PythonizeNamedMappingType for NamedCustomDict { + type Builder<'py> = Bound<'py, NamedCustomDict>; - fn builder(py: Python<'py>, len: usize, name: &'static str) -> PyResult { + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult> { Bound::new( py, NamedCustomDict { @@ -228,21 +232,21 @@ impl<'py> PythonizeNamedMappingType<'py> for NamedCustomDict { ) } - fn push_field( - builder: &mut Self::Builder, + fn push_field<'py>( + builder: &mut Self::Builder<'py>, name: Bound<'py, pyo3::types::PyString>, value: Bound<'py, PyAny>, ) -> PyResult<()> { unsafe { builder.downcast_unchecked::() }.set_item(name, value) } - fn finish(builder: Self::Builder) -> PyResult> { + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { Ok(unsafe { builder.into_any().downcast_into_unchecked() }) } } struct PythonizeNamedCustomDict; -impl<'py> PythonizeTypes<'py> for PythonizeNamedCustomDict { +impl<'py> PythonizeTypes for PythonizeNamedCustomDict { type Map = CustomDict; type NamedMap = NamedCustomDict; type List = PyTuple; diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 5f2b970..9c3b688 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -13,9 +13,9 @@ struct Root { root_map: BTreeMap>, } -impl<'py, T> PythonizeTypes<'py> for Root { +impl<'py, T> PythonizeTypes for Root { type Map = PyDict; - type NamedMap = PythonizeUnnamedMappingAdapter<'py, PyDict>; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyList; } From bc3caf522f8d4ddfceda8c6ca438b5e6ff16972d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 30 Aug 2025 10:19:23 +0100 Subject: [PATCH 74/77] release: 0.26 (#95) * release: 0.26 * use trusted publishing for release --- .github/workflows/release.yml | 19 +++++++--- CHANGELOG.md | 14 ++++++++ Cargo.toml | 8 ++--- README.md | 2 +- src/de.rs | 40 ++++++++++----------- src/error.rs | 2 +- src/ser.rs | 4 +-- tests/test_custom_types.rs | 52 +++++++++++++--------------- tests/test_with_serde_path_to_err.rs | 10 +++--- 9 files changed, 87 insertions(+), 64 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f21b46..f538d3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,18 +4,29 @@ on: push: tags: - "v*" + workflow_dispatch: + inputs: + version: + description: The version to build jobs: release: + permissions: + id-token: write + runs-on: ubuntu-latest environment: release steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + # The tag to build or the tag received by the tag event + ref: ${{ github.event.inputs.version || github.ref }} + persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: rust-lang/crates-io-auth-action@v1 + id: auth - name: Publish to crates.io run: cargo publish env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 16b1080..a21bd55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 0.26.0 - 2025-08-30 + +### Packaging +- Bump MSRV to 1.74 +- Update to PyO3 0.26 + +### Changed +- `PythonizeTypes`, `PythonizeMappingType` and `PythonizeNamedMappingType` no longer have a lifetime on the trait, instead the `Builder` type is a GAT. + +## 0.25.0 - 2025-05-23 + +### Packaging +- Update to PyO3 0.25 + ## 0.24.0 - 2025-03-26 ### Packaging diff --git a/Cargo.toml b/Cargo.toml index d636384..c0797e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "pythonize" -version = "0.25.0" +version = "0.26.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" -rust-version = "1.65" +rust-version = "1.74" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.25", default-features = false } +pyo3 = { version = "0.26", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.25", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.26", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" diff --git a/README.md b/README.md index 441f29f..2667523 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ let sample = Sample { bar: None }; -Python::with_gil(|py| { +Python::attach(|py| { // Rust -> Python let obj = pythonize(py, &sample).unwrap(); diff --git a/src/de.rs b/src/de.rs index 2860e48..a30dbca 100644 --- a/src/de.rs +++ b/src/de.rs @@ -24,7 +24,7 @@ impl<'a, 'py> Depythonizer<'a, 'py> { } fn sequence_access(&self, expected_len: Option) -> Result> { - let seq = self.input.downcast::()?; + let seq = self.input.cast::()?; let len = self.input.len()?; match expected_len { @@ -36,10 +36,10 @@ impl<'a, 'py> Depythonizer<'a, 'py> { } fn set_access(&self) -> Result> { - match self.input.downcast::() { + match self.input.cast::() { Ok(set) => Ok(PySetAsSequence::from_set(set)), Err(e) => { - if let Ok(f) = self.input.downcast::() { + if let Ok(f) = self.input.cast::() { Ok(PySetAsSequence::from_frozenset(f)) } else { Err(e.into()) @@ -49,7 +49,7 @@ impl<'a, 'py> Depythonizer<'a, 'py> { } fn dict_access(&self) -> Result> { - PyMappingAccess::new(self.input.downcast()?) + PyMappingAccess::new(self.input.cast()?) } fn deserialize_any_int<'de, V>(&self, int: &Bound<'_, PyInt>, visitor: V) -> Result @@ -111,7 +111,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { self.deserialize_unit(visitor) } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if let Ok(x) = obj.downcast::() { + } else if let Ok(x) = obj.cast::() { self.deserialize_any_int(x, visitor) } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) @@ -128,9 +128,9 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { self.deserialize_f64(visitor) } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_seq(visitor) - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { self.deserialize_map(visitor) } else { Err(obj.get_type().qualname().map_or_else( @@ -151,7 +151,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - let s = self.input.downcast::()?.to_cow()?; + let s = self.input.cast::()?.to_cow()?; if s.len() != 1 { return Err(PythonizeError::invalid_length_char()); } @@ -175,7 +175,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - let s = self.input.downcast::()?; + let s = self.input.cast::()?; visitor.visit_str(&s.to_cow()?) } @@ -190,7 +190,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { where V: de::Visitor<'de>, { - let b = self.input.downcast::()?; + let b = self.input.cast::()?; visitor.visit_bytes(b.as_bytes()) } @@ -303,9 +303,9 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { V: de::Visitor<'de>, { let item = &self.input; - if let Ok(s) = item.downcast::() { + if let Ok(s) = item.cast::() { visitor.visit_enum(s.to_cow()?.into_deserializer()) - } else if let Ok(m) = item.downcast::() { + } else if let Ok(m) = item.cast::() { // Get the enum variant from the mapping key if m.len()? != 1 { return Err(PythonizeError::invalid_length_enum()); @@ -313,7 +313,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { let variant: Bound = m .keys()? .get_item(0)? - .downcast_into::() + .cast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; let value = m.get_item(&variant)?; visitor.visit_enum(PyEnumAccess::new(&value, variant)) @@ -328,7 +328,7 @@ impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { { let s = self .input - .downcast::() + .cast::() .map_err(|_| PythonizeError::dict_key_not_string())?; visitor.visit_str(&s.to_cow()?) } @@ -528,7 +528,7 @@ mod test { where T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { - Python::with_gil(|py| { + Python::attach(|py| { let obj = py.eval(code, None, None).unwrap(); let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); @@ -585,7 +585,7 @@ mod test { let code = c_str!("{'foo': 'Foo'}"); - Python::with_gil(|py| { + Python::attach(|py| { let locals = PyDict::new(py); let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( @@ -613,7 +613,7 @@ mod test { let code = c_str!("('cat', -10.05, 'foo')"); - Python::with_gil(|py| { + Python::attach(|py| { let locals = PyDict::new(py); let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( @@ -825,7 +825,7 @@ mod test { #[test] fn test_int_limits() { - Python::with_gil(|py| { + Python::attach(|py| { // serde_json::Value supports u64 and i64 as maximum sizes let _: serde_json::Value = depythonize(&u8::MAX.into_pyobject(py).unwrap()).unwrap(); let _: serde_json::Value = depythonize(&u8::MIN.into_pyobject(py).unwrap()).unwrap(); @@ -857,7 +857,7 @@ mod test { #[test] fn test_deserialize_bytes() { - Python::with_gil(|py| { + Python::attach(|py| { let obj = PyBytes::new(py, "hello".as_bytes()); let actual: Vec = depythonize(&obj).unwrap(); assert_eq!(actual, b"hello"); @@ -874,7 +874,7 @@ mod test { #[test] fn test_unknown_type() { - Python::with_gil(|py| { + Python::attach(|py| { let obj = py .import("decimal") .unwrap() diff --git a/src/error.rs b/src/error.rs index 1bcc556..7828a71 100644 --- a/src/error.rs +++ b/src/error.rs @@ -73,7 +73,7 @@ pub enum ErrorImpl { Message(String), /// A Python type not supported by the deserializer UnsupportedType(String), - /// A `PyAny` object that failed to downcast to an expected Python type + /// A `PyAny` object that failed to cast to an expected Python type UnexpectedType(String), /// Dict keys should be strings to deserialize to struct fields DictKeyNotString, diff --git a/src/ser.rs b/src/ser.rs index dce22e3..c8e6dd1 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -632,7 +632,7 @@ mod test { where T: Serialize, { - Python::with_gil(|py| -> PyResult<()> { + Python::attach(|py| -> PyResult<()> { let obj = pythonize(py, &src)?; let locals = PyDict::new(py); @@ -845,7 +845,7 @@ mod test { // serde treats &[u8] as a sequence of integers due to lack of specialization test_ser(b"foo", "[102,111,111]"); - Python::with_gil(|py| { + Python::attach(|py| { assert!(pythonize(py, serde_bytes::Bytes::new(b"foo")) .expect("bytes will always serialize successfully") .eq(&PyBytes::new(py, b"foo")) diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 32c5768..27888d0 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -4,7 +4,7 @@ use pyo3::{ exceptions::{PyIndexError, PyKeyError}, prelude::*, types::{PyDict, PyMapping, PySequence, PyTuple}, - BoundObject, + IntoPyObjectExt, }; use pythonize::{ depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, @@ -15,7 +15,7 @@ use serde_json::{json, Value}; #[pyclass(sequence)] struct CustomList { - items: Vec, + items: Vec>, } #[pymethods] @@ -24,7 +24,7 @@ impl CustomList { self.items.len() } - fn __getitem__(&self, idx: isize) -> PyResult { + fn __getitem__(&self, idx: isize) -> PyResult> { self.items .get(idx as usize) .cloned() @@ -46,14 +46,12 @@ impl PythonizeListType for CustomList { CustomList { items: elements .into_iter() - .map(|item| item.into_pyobject(py).map(|x| x.into_any().unbind())) - .collect::, T::Error>>() - .map_err(Into::into)?, + .map(|item| item.into_py_any(py)) + .collect::>()?, }, - )? - .into_any(); + )?; - Ok(unsafe { sequence.downcast_into_unchecked() }) + Ok(unsafe { sequence.cast_into_unchecked() }) } } @@ -66,7 +64,7 @@ impl<'py> PythonizeTypes for PythonizeCustomList { #[test] fn test_custom_list() { - Python::with_gil(|py| { + Python::attach(|py| { PySequence::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!([1, 2, 3])).unwrap(); assert!(serialized.is_instance_of::()); @@ -78,7 +76,7 @@ fn test_custom_list() { #[pyclass(mapping)] struct CustomDict { - items: HashMap, + items: HashMap>, } #[pymethods] @@ -87,14 +85,14 @@ impl CustomDict { self.items.len() } - fn __getitem__(&self, key: String) -> PyResult { + fn __getitem__(&self, key: String) -> PyResult> { self.items .get(&key) .cloned() .ok_or_else(|| PyKeyError::new_err(key)) } - fn __setitem__(&mut self, key: String, value: PyObject) { + fn __setitem__(&mut self, key: String, value: Py) { self.items.insert(key, value); } @@ -102,7 +100,7 @@ impl CustomDict { self.items.keys().collect() } - fn values(&self) -> Vec { + fn values(&self) -> Vec> { self.items.values().cloned().collect() } } @@ -124,11 +122,11 @@ impl PythonizeMappingType for CustomDict { key: Bound<'py, PyAny>, value: Bound<'py, PyAny>, ) -> PyResult<()> { - unsafe { builder.downcast_unchecked::() }.set_item(key, value) + unsafe { builder.cast_unchecked::() }.set_item(key, value) } fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { - Ok(unsafe { builder.into_any().downcast_into_unchecked() }) + Ok(unsafe { builder.cast_into_unchecked() }) } } @@ -141,7 +139,7 @@ impl<'py> PythonizeTypes for PythonizeCustomDict { #[test] fn test_custom_dict() { - Python::with_gil(|py| { + Python::attach(|py| { PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) @@ -155,7 +153,7 @@ fn test_custom_dict() { #[test] fn test_tuple() { - Python::with_gil(|py| { + Python::attach(|py| { PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!([1, 2, 3, 4])).unwrap(); @@ -169,7 +167,7 @@ fn test_tuple() { #[test] fn test_pythonizer_can_be_created() { // https://github.com/davidhewitt/pythonize/pull/56 - Python::with_gil(|py| { + Python::attach(|py| { let sample = json!({ "hello": 1, "world": 2 }); assert!(sample .serialize(Pythonizer::new(py)) @@ -186,7 +184,7 @@ fn test_pythonizer_can_be_created() { #[pyclass(mapping)] struct NamedCustomDict { name: String, - items: HashMap, + items: HashMap>, } #[pymethods] @@ -195,14 +193,14 @@ impl NamedCustomDict { self.items.len() } - fn __getitem__(&self, key: String) -> PyResult { + fn __getitem__(&self, key: String) -> PyResult> { self.items .get(&key) .cloned() .ok_or_else(|| PyKeyError::new_err(key)) } - fn __setitem__(&mut self, key: String, value: PyObject) { + fn __setitem__(&mut self, key: String, value: Py) { self.items.insert(key, value); } @@ -210,7 +208,7 @@ impl NamedCustomDict { self.items.keys().collect() } - fn values(&self) -> Vec { + fn values(&self) -> Vec> { self.items.values().cloned().collect() } } @@ -237,11 +235,11 @@ impl PythonizeNamedMappingType for NamedCustomDict { name: Bound<'py, pyo3::types::PyString>, value: Bound<'py, PyAny>, ) -> PyResult<()> { - unsafe { builder.downcast_unchecked::() }.set_item(name, value) + unsafe { builder.cast_unchecked::() }.set_item(name, value) } fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { - Ok(unsafe { builder.into_any().downcast_into_unchecked() }) + Ok(unsafe { builder.cast_into_unchecked() }) } } @@ -260,7 +258,7 @@ struct Struct { #[test] fn test_custom_unnamed_dict() { - Python::with_gil(|py| { + Python::attach(|py| { PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &Struct { hello: 1, world: 2 }).unwrap(); @@ -273,7 +271,7 @@ fn test_custom_unnamed_dict() { #[test] fn test_custom_named_dict() { - Python::with_gil(|py| { + Python::attach(|py| { PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &Struct { hello: 1, world: 2 }) diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index 9c3b688..d92c4d1 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -40,7 +40,7 @@ impl Serialize for CannotSerialize { #[test] fn test_de_valid() { - Python::with_gil(|py| { + Python::attach(|py| { let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); @@ -82,7 +82,7 @@ fn test_de_valid() { #[test] fn test_de_invalid() { - Python::with_gil(|py| { + Python::attach(|py| { let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); @@ -106,7 +106,7 @@ fn test_de_invalid() { #[test] fn test_ser_valid() { - Python::with_gil(|py| { + Python::attach(|py| { let root = Root { root_key: String::from("root_value"), root_map: BTreeMap::from([ @@ -128,7 +128,7 @@ fn test_ser_valid() { let ser = pythonize::Pythonizer::>::from(py); let pyroot: Bound<'_, PyAny> = serde_path_to_error::serialize(&root, ser).unwrap(); - let pyroot = pyroot.downcast::().unwrap(); + let pyroot = pyroot.cast::().unwrap(); assert_eq!(pyroot.len(), 2); let root_value: String = pyroot @@ -181,7 +181,7 @@ fn test_ser_valid() { #[test] fn test_ser_invalid() { - Python::with_gil(|py| { + Python::attach(|py| { let root = Root { root_key: String::from("root_value"), root_map: BTreeMap::from([ From 9d6d9287693032b869909bfe8dd07a8341749dd9 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 7 Nov 2025 21:50:18 +0000 Subject: [PATCH 75/77] ci: extend test matrix to 3.14, more arm runners (#98) --- .github/workflows/ci.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c08531..3d69440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,17 +49,21 @@ jobs: strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: - python-architecture: ["x64"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] - os: ["macos-13", "ubuntu-latest", "windows-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + os: ["macos-latest", "ubuntu-latest", "windows-latest"] rust: [stable] include: - - python-version: "3.13" + - python-version: "3.14" os: "ubuntu-latest" rust: ${{ needs.resolve.outputs.MSRV }} - - python-version: "3.13" - python-architecture: "arm64" - os: "macos-latest" + - python-version: "3.14" + os: "macos-15-intel" + rust: "stable" + - python-version: "3.14" + os: "ubuntu-24.04-arm" + rust: "stable" + - python-version: "3.14" + os: "windows-11-arm" rust: "stable" steps: @@ -69,7 +73,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.python-python-architecture }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -88,9 +91,7 @@ jobs: - name: Test run: cargo test --verbose - # https://github.com/PyO3/pyo3/issues/4709 - can't use abi3 w. freethreaded build - - if: ${{ !endsWith(matrix.python-version, 't') }} - name: Test (abi3) + - name: Test (abi3) run: cargo test --verbose --features pyo3/abi3-py37 env: From 89420b2d91721f277f073befa70a95016606e9ed Mon Sep 17 00:00:00 2001 From: Tino Wagner Date: Fri, 7 Nov 2025 22:50:36 +0100 Subject: [PATCH 76/77] Bump PyO3 to 0.27 (#96) * Bump PyO3 to 0.27 Update pyo3 dependency to 0.27. Replace deprecated DowncastError with CastError. --------- Co-authored-by: David Hewitt --- Cargo.toml | 4 ++-- src/error.rs | 14 +++++++------- tests/test_with_serde_path_to_err.rs | 11 +++++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c0797e7..ce406ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.26", default-features = false } +pyo3 = { version = "0.27", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.26", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.27", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" diff --git a/src/error.rs b/src/error.rs index 7828a71..b608106 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,5 @@ use pyo3::PyErr; -use pyo3::{exceptions::*, DowncastError, DowncastIntoError}; +use pyo3::{exceptions::*, CastError, CastIntoError}; use serde::{de, ser}; use std::convert::Infallible; use std::error; @@ -153,18 +153,18 @@ impl From for PythonizeError { } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'a, 'py> From> for PythonizeError { - fn from(other: DowncastError<'a, 'py>) -> Self { +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'a, 'py> From> for PythonizeError { + fn from(other: CastError<'a, 'py>) -> Self { Self { inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), } } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'py> From> for PythonizeError { - fn from(other: DowncastIntoError<'py>) -> Self { +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'py> From> for PythonizeError { + fn from(other: CastIntoError<'py>) -> Self { Self { inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index d92c4d1..82fd8bb 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -100,7 +100,10 @@ fn test_de_invalid() { let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); - assert_eq!(err.to_string(), "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be converted to 'PyString'"); + assert_eq!( + err.to_string(), + "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be cast as 'str'" + ); }) } @@ -143,7 +146,7 @@ fn test_ser_valid() { .get_item("root_map") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(root_map.len(), 2); @@ -151,7 +154,7 @@ fn test_ser_valid() { .get_item("nested_0") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(nested_0.len(), 1); let nested_key_0: String = nested_0 @@ -166,7 +169,7 @@ fn test_ser_valid() { .get_item("nested_1") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(nested_1.len(), 1); let nested_key_1: String = nested_1 From 43c714f13d40db35b080659c67b818758cc8c202 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 7 Nov 2025 21:57:31 +0000 Subject: [PATCH 77/77] release: 0.27.0 (#99) --- CHANGELOG.md | 3 +++ Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a21bd55..f267740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.27.0 - 2025-11-07 +- Update to PyO3 0.27 + ## 0.26.0 - 2025-08-30 ### Packaging diff --git a/Cargo.toml b/Cargo.toml index ce406ad..4a714a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pythonize" -version = "0.26.0" +version = "0.27.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" rust-version = "1.74"