diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62b3e2d..3d69440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,19 @@ env: CARGO_TERM_COLOR: always jobs: + resolve: + runs-on: ubuntu-latest + outputs: + MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: resolve MSRV + id: resolve-msrv + run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT + fmt: runs-on: ubuntu-latest steps: @@ -30,27 +43,27 @@ jobs: - run: cargo clippy --all build: - needs: [fmt] # don't wait for clippy as fails rarely and takes longer + needs: [resolve, fmt] # don't wait for clippy as fails rarely and takes longer name: python${{ matrix.python-version }} ${{ matrix.os }} rust-${{ matrix.rust}} runs-on: ${{ matrix.os }} strategy: - fail-fast: false # If one platform fails, allow the rest to keep testing. + fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: - python-architecture: ["x64"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ - "macos-13", - "ubuntu-latest", - "windows-latest", - ] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + os: ["macos-latest", "ubuntu-latest", "windows-latest"] rust: [stable] include: - - python-version: "3.12" + - python-version: "3.14" os: "ubuntu-latest" - rust: "1.63" - - python-version: "3.12" - python-architecture: "arm64" - os: "macos-latest" + rust: ${{ needs.resolve.outputs.MSRV }} + - python-version: "3.14" + os: "macos-15-intel" + rust: "stable" + - python-version: "3.14" + os: "ubuntu-24.04-arm" + rust: "stable" + - python-version: "3.14" + os: "windows-11-arm" rust: "stable" steps: @@ -60,7 +73,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.python-python-architecture }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -70,6 +82,12 @@ jobs: - uses: Swatinem/rust-cache@v2 continue-on-error: true + - if: ${{ matrix.rust == needs.resolve.outputs.MSRV }} + name: Set dependencies on MSRV + run: cargo +stable update + env: + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback + - name: Test run: cargo test --verbose diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f538d3d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release Rust Crate + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: The version to build + +jobs: + release: + permissions: + id-token: write + + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@v5 + with: + # The tag to build or the tag received by the tag event + ref: ${{ github.event.inputs.version || github.ref }} + persist-credentials: false + + - uses: rust-lang/crates-io-auth-action@v1 + id: auth + + - name: Publish to crates.io + run: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f245889..f267740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,58 @@ -## Unreleased +## 0.27.0 - 2025-11-07 +- Update to PyO3 0.27 +## 0.26.0 - 2025-08-30 + +### Packaging +- Bump MSRV to 1.74 +- Update to PyO3 0.26 + +### Changed +- `PythonizeTypes`, `PythonizeMappingType` and `PythonizeNamedMappingType` no longer have a lifetime on the trait, instead the `Builder` type is a GAT. + +## 0.25.0 - 2025-05-23 + +### Packaging +- Update to PyO3 0.25 + +## 0.24.0 - 2025-03-26 + +### Packaging +- Update to PyO3 0.24 + +## Removed +- Remove deprecated `depythonize_bound()` + +## 0.23.0 - 2024-11-22 + +### Packaging +- Update to PyO3 0.23 + +## 0.22.0 - 2024-08-10 + +### Packaging - Bump MSRV to 1.63 - Update to PyO3 0.22 + +### Added - Support `u128` / `i128` integers. -- Remove support for PyO3's `gil-refs` feature +- Implement `PythonizeListType` for `PyTuple` +- Support deserializing enums from any `PyMapping` instead of just `PyDict` +- Support serializing struct-like types to named mappings using `PythonizeTypes::NamedMap` + +### Changed - `pythonize()` now returns `Bound<'py, PyAny>` instead of `Py` -- `depythonize()` now take a `&Bound` and is no longer deprecated +- `depythonize()` now take `&'a Bound` and is no longer deprecated - `depythonize_bound()` is now deprecated -- `Depythonizer` now contains a `&Bound` and so has an extra lifetime `'bound` -- `Depythonizer::from_object()` now takes a `&Bound` and is no longer deprecated +- `Depythonizer::from_object()` now takes `&'a Bound` and is no longer deprecated +- `Depythonizer` now contains `&'a Bound` and so has an extra lifetime `'a` + +### Removed +- Remove support for PyO3's `gil-refs` feature + +### Fixed - Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` +- Fix deserializing `set` and `frozenset` into Rust homogeneous containers ## 0.21.1 - 2024-04-02 diff --git a/Cargo.toml b/Cargo.toml index 645ca32..4a714a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "pythonize" -version = "0.21.1" +version = "0.27.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] edition = "2021" -rust-version = "1.63" +rust-version = "1.74" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" @@ -13,11 +13,11 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.22.0", default-features = false } +pyo3 = { version = "0.27", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.22.0", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } +pyo3 = { version = "0.27", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" serde_bytes = "0.11" maplit = "1.0.2" diff --git a/README.md b/README.md index 624e22d..2667523 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,7 @@ that which is produced directly by `pythonize`. This crate converts Rust types which implement the [Serde] serialization traits into Python objects using the [PyO3] library. -Pythonize has two public APIs: `pythonize` and `depythonize_bound`. - - -
- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`, and pythonize is doing the same. +Pythonize has two main public APIs: `pythonize` and `depythonize`.
@@ -28,8 +21,8 @@ PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the ```rust use serde::{Serialize, Deserialize}; -use pyo3::Python; -use pythonize::{depythonize_bound, pythonize}; +use pyo3::prelude::*; +use pythonize::{depythonize, pythonize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] struct Sample { @@ -37,21 +30,20 @@ struct Sample { bar: Option } -let gil = Python::acquire_gil(); -let py = gil.python(); - let sample = Sample { foo: "Foo".to_string(), bar: None }; -// Rust -> Python -let obj = pythonize(py, &sample).unwrap(); +Python::attach(|py| { + // Rust -> Python + let obj = pythonize(py, &sample).unwrap(); -assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); + assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.repr().unwrap())); -// Python -> Rust -let new_sample: Sample = depythonize_bound(obj.into_bound(py)).unwrap(); + // Python -> Rust + let new_sample: Sample = depythonize(&obj).unwrap(); -assert_eq!(new_sample, sample); + assert_eq!(new_sample, sample); +}) ``` diff --git a/src/de.rs b/src/de.rs index 11d0fcc..a30dbca 100644 --- a/src/de.rs +++ b/src/de.rs @@ -2,43 +2,29 @@ use pyo3::{types::*, Bound}; use serde::de::{self, IntoDeserializer}; use serde::Deserialize; -use crate::error::{PythonizeError, Result}; +use crate::error::{ErrorImpl, PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` -pub fn depythonize<'py, 'obj, T>(obj: &'obj Bound<'py, PyAny>) -> Result +pub fn depythonize<'a, 'py, T>(obj: &'a Bound<'py, PyAny>) -> Result where - T: Deserialize<'obj>, + T: Deserialize<'a>, { - let mut depythonizer = Depythonizer::from_object(obj); - T::deserialize(&mut depythonizer) -} - -/// Attempt to convert a Python object to an instance of `T` -#[deprecated(since = "0.22.0", note = "use `depythonize` instead")] -pub fn depythonize_bound<'py, T>(obj: Bound<'py, PyAny>) -> Result -where - T: for<'a> Deserialize<'a>, -{ - let mut depythonizer = Depythonizer::from_object(&obj); - T::deserialize(&mut depythonizer) + T::deserialize(&mut Depythonizer::from_object(obj)) } /// A structure that deserializes Python objects into Rust values -pub struct Depythonizer<'py, 'bound> { - input: &'bound Bound<'py, PyAny>, +pub struct Depythonizer<'a, 'py> { + input: &'a Bound<'py, PyAny>, } -impl<'py, 'bound> Depythonizer<'py, 'bound> { +impl<'a, 'py> Depythonizer<'a, 'py> { /// Create a deserializer from a Python object - pub fn from_object<'input>(input: &'input Bound<'py, PyAny>) -> Depythonizer<'py, 'input> { + pub fn from_object(input: &'a Bound<'py, PyAny>) -> Self { Depythonizer { input } } - fn sequence_access( - &self, - expected_len: Option, - ) -> Result> { - let seq = self.input.downcast::()?; + fn sequence_access(&self, expected_len: Option) -> Result> { + let seq = self.input.cast::()?; let len = self.input.len()?; match expected_len { @@ -49,8 +35,21 @@ impl<'py, 'bound> Depythonizer<'py, 'bound> { } } + fn set_access(&self) -> Result> { + match self.input.cast::() { + Ok(set) => Ok(PySetAsSequence::from_set(set)), + Err(e) => { + if let Ok(f) = self.input.cast::() { + Ok(PySetAsSequence::from_frozenset(f)) + } else { + Err(e.into()) + } + } + } + } + fn dict_access(&self) -> Result> { - PyMappingAccess::new(self.input.downcast()?) + PyMappingAccess::new(self.input.cast()?) } fn deserialize_any_int<'de, V>(&self, int: &Bound<'_, PyInt>, visitor: V) -> Result @@ -97,14 +96,14 @@ macro_rules! deserialize_type { }; } -impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, 'bound> { +impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { type Error = PythonizeError; fn deserialize_any(self, visitor: V) -> Result where V: de::Visitor<'de>, { - let obj = &self.input; + let obj = self.input; // First check for cases which are cheap to check due to pointer // comparison or bitflag checks @@ -112,7 +111,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' self.deserialize_unit(visitor) } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if let Ok(x) = obj.downcast::() { + } else if let Ok(x) = obj.cast::() { self.deserialize_any_int(x, visitor) } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) @@ -122,17 +121,16 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' self.deserialize_str(visitor) } // Continue with cases which are slower to check because they go - // throuh `isinstance` machinery + // through `isinstance` machinery else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_bytes(visitor) } else if obj.is_instance_of::() { self.deserialize_f64(visitor) - } else if obj.is_instance_of::() - || obj.is_instance_of::() - || obj.downcast::().is_ok() - { + } else if obj.is_instance_of::() || obj.is_instance_of::() { + self.deserialize_seq(visitor) + } else if obj.cast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { self.deserialize_map(visitor) } else { Err(obj.get_type().qualname().map_or_else( @@ -153,7 +151,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' where V: de::Visitor<'de>, { - let s = self.input.downcast::()?.to_cow()?; + let s = self.input.cast::()?.to_cow()?; if s.len() != 1 { return Err(PythonizeError::invalid_length_char()); } @@ -177,7 +175,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' where V: de::Visitor<'de>, { - let s = self.input.downcast::()?; + let s = self.input.cast::()?; visitor.visit_str(&s.to_cow()?) } @@ -192,7 +190,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' where V: de::Visitor<'de>, { - let b = self.input.downcast::()?; + let b = self.input.cast::()?; visitor.visit_bytes(b.as_bytes()) } @@ -243,7 +241,18 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' where V: de::Visitor<'de>, { - visitor.visit_seq(self.sequence_access(None)?) + match self.sequence_access(None) { + Ok(seq) => visitor.visit_seq(seq), + Err(e) => { + // we allow sets to be deserialized as sequences, so try that + if matches!(*e.inner, ErrorImpl::UnexpectedType(_)) { + if let Ok(set) = self.set_access() { + return visitor.visit_seq(set); + } + } + Err(e) + } + } } fn deserialize_tuple(self, len: usize, visitor: V) -> Result @@ -294,21 +303,20 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' V: de::Visitor<'de>, { let item = &self.input; - if let Ok(d) = item.downcast::() { - // Get the enum variant from the dict key - if d.len() != 1 { + if let Ok(s) = item.cast::() { + visitor.visit_enum(s.to_cow()?.into_deserializer()) + } else if let Ok(m) = item.cast::() { + // Get the enum variant from the mapping key + if m.len()? != 1 { return Err(PythonizeError::invalid_length_enum()); } - let variant = d - .keys() + let variant: Bound = m + .keys()? .get_item(0)? - .downcast_into::() + .cast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; - let value = d.get_item(&variant)?.unwrap(); - let mut de = Depythonizer::from_object(&value); - visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) - } else if let Ok(s) = item.downcast::() { - visitor.visit_enum(s.to_cow()?.into_deserializer()) + let value = m.get_item(&variant)?; + visitor.visit_enum(PyEnumAccess::new(&value, variant)) } else { Err(PythonizeError::invalid_enum_type()) } @@ -320,7 +328,7 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' { let s = self .input - .downcast::() + .cast::() .map_err(|_| PythonizeError::dict_key_not_string())?; visitor.visit_str(&s.to_cow()?) } @@ -333,19 +341,19 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' } } -struct PySequenceAccess<'py, 'bound> { - seq: &'bound Bound<'py, PySequence>, +struct PySequenceAccess<'a, 'py> { + seq: &'a Bound<'py, PySequence>, index: usize, len: usize, } -impl<'py, 'bound> PySequenceAccess<'py, 'bound> { - fn new(seq: &'bound Bound<'py, PySequence>, len: usize) -> Self { +impl<'a, 'py> PySequenceAccess<'a, 'py> { + fn new(seq: &'a Bound<'py, PySequence>, len: usize) -> Self { Self { seq, index: 0, len } } } -impl<'de, 'py, 'bound> de::SeqAccess<'de> for PySequenceAccess<'py, 'bound> { +impl<'de> de::SeqAccess<'de> for PySequenceAccess<'_, '_> { type Error = PythonizeError; fn next_element_seed(&mut self, seed: T) -> Result> @@ -354,18 +362,52 @@ impl<'de, 'py, 'bound> de::SeqAccess<'de> for PySequenceAccess<'py, 'bound> { { if self.index < self.len { let item = self.seq.get_item(self.index)?; - let mut item_de = Depythonizer::from_object(&item); self.index += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } } } +struct PySetAsSequence<'py> { + iter: Bound<'py, PyIterator>, +} + +impl<'py> PySetAsSequence<'py> { + fn from_set(set: &Bound<'py, PySet>) -> Self { + Self { + iter: PyIterator::from_object(set).expect("set is always iterable"), + } + } + + fn from_frozenset(set: &Bound<'py, PyFrozenSet>) -> Self { + Self { + iter: PyIterator::from_object(set).expect("frozenset is always iterable"), + } + } +} + +impl<'de> de::SeqAccess<'de> for PySetAsSequence<'_> { + type Error = PythonizeError; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: de::DeserializeSeed<'de>, + { + match self.iter.next() { + Some(item) => seed + .deserialize(&mut Depythonizer::from_object(&item?)) + .map(Some), + None => Ok(None), + } + } +} + struct PyMappingAccess<'py> { - keys: Bound<'py, PySequence>, - values: Bound<'py, PySequence>, + keys: Bound<'py, PyList>, + values: Bound<'py, PyList>, key_idx: usize, val_idx: usize, len: usize, @@ -386,7 +428,7 @@ impl<'py> PyMappingAccess<'py> { } } -impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { +impl<'de> de::MapAccess<'de> for PyMappingAccess<'_> { type Error = PythonizeError; fn next_key_seed(&mut self, seed: K) -> Result> @@ -395,9 +437,9 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { { if self.key_idx < self.len { let item = self.keys.get_item(self.key_idx)?; - let mut item_de = Depythonizer::from_object(&item); self.key_idx += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } @@ -408,24 +450,26 @@ impl<'de, 'py> de::MapAccess<'de> for PyMappingAccess<'py> { V: de::DeserializeSeed<'de>, { let item = self.values.get_item(self.val_idx)?; - let mut item_de = Depythonizer::from_object(&item); self.val_idx += 1; - seed.deserialize(&mut item_de) + seed.deserialize(&mut Depythonizer::from_object(&item)) } } -struct PyEnumAccess<'a, 'py, 'bound> { - de: &'a mut Depythonizer<'py, 'bound>, +struct PyEnumAccess<'a, 'py> { + de: Depythonizer<'a, 'py>, variant: Bound<'py, PyString>, } -impl<'a, 'py, 'bound> PyEnumAccess<'a, 'py, 'bound> { - fn new(de: &'a mut Depythonizer<'py, 'bound>, variant: Bound<'py, PyString>) -> Self { - Self { de, variant } +impl<'a, 'py> PyEnumAccess<'a, 'py> { + fn new(obj: &'a Bound<'py, PyAny>, variant: Bound<'py, PyString>) -> Self { + Self { + de: Depythonizer::from_object(obj), + variant, + } } } -impl<'a, 'py, 'de, 'bound> de::EnumAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { +impl<'de> de::EnumAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; type Variant = Self; @@ -440,7 +484,7 @@ impl<'a, 'py, 'de, 'bound> de::EnumAccess<'de> for PyEnumAccess<'a, 'py, 'bound> } } -impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bound> { +impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; fn unit_variant(self) -> Result<()> { @@ -451,7 +495,7 @@ impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bou where T: de::DeserializeSeed<'de>, { - seed.deserialize(self.de) + seed.deserialize(&mut { self.de }) } fn tuple_variant(self, len: usize, visitor: V) -> Result @@ -471,23 +515,24 @@ impl<'a, 'py, 'de, 'bound> de::VariantAccess<'de> for PyEnumAccess<'a, 'py, 'bou #[cfg(test)] mod test { + use std::ffi::CStr; + use super::*; use crate::error::ErrorImpl; use maplit::hashmap; - use pyo3::{IntoPy, Python}; + use pyo3::ffi::c_str; + use pyo3::{IntoPyObject, Python}; use serde_json::{json, Value as JsonValue}; - fn test_de(code: &str, expected: &T, expected_json: &JsonValue) + fn test_de(code: &CStr, expected: &T, expected_json: &JsonValue) where T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { - Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + Python::attach(|py| { + let obj = py.eval(code, None, None).unwrap(); let actual: T = depythonize(&obj).unwrap(); assert_eq!(&actual, expected); + let actual_json: JsonValue = depythonize(&obj).unwrap(); assert_eq!(&actual_json, expected_json); }); @@ -500,7 +545,7 @@ mod test { let expected = Empty; let expected_json = json!(null); - let code = "None"; + let code = c_str!("None"); test_de(code, &expected, &expected_json); } @@ -526,7 +571,7 @@ mod test { "baz": 45.23, "qux": true }); - let code = "{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"; + let code = c_str!("{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"); test_de(code, &expected, &expected_json); } @@ -538,13 +583,11 @@ mod test { bar: usize, } - let code = "{'foo': 'Foo'}"; + let code = c_str!("{'foo': 'Foo'}"); - Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + Python::attach(|py| { + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( *depythonize::(&obj).unwrap_err().inner, ErrorImpl::Message(msg) if msg == "missing field `bar`" @@ -559,7 +602,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "('cat', -10.05)"; + let code = c_str!("('cat', -10.05)"); test_de(code, &expected, &expected_json); } @@ -568,13 +611,11 @@ mod test { #[derive(Debug, Deserialize, PartialEq)] struct TupleStruct(String, f64); - let code = "('cat', -10.05, 'foo')"; + let code = c_str!("('cat', -10.05, 'foo')"); - Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound(&format!("obj = {}", code), None, Some(&locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap().unwrap(); + Python::attach(|py| { + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); assert!(matches!( *depythonize::(&obj).unwrap_err().inner, ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 @@ -589,7 +630,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "['cat', -10.05]"; + let code = c_str!("['cat', -10.05]"); test_de(code, &expected, &expected_json); } @@ -597,7 +638,7 @@ mod test { fn test_tuple() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "('foo', 5)"; + let code = c_str!("('foo', 5)"); test_de(code, &expected, &expected_json); } @@ -605,7 +646,23 @@ mod test { fn test_tuple_from_pylist() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "['foo', 5]"; + let code = c_str!("['foo', 5]"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_vec_from_pyset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = c_str!("{'foo'}"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_vec_from_pyfrozenset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = c_str!("frozenset({'foo'})"); test_de(code, &expected, &expected_json); } @@ -613,7 +670,7 @@ mod test { fn test_vec() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "[3, 2, 1]"; + let code = c_str!("[3, 2, 1]"); test_de(code, &expected, &expected_json); } @@ -621,7 +678,7 @@ mod test { fn test_vec_from_tuple() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "(3, 2, 1)"; + let code = c_str!("(3, 2, 1)"); test_de(code, &expected, &expected_json); } @@ -629,7 +686,7 @@ mod test { fn test_hashmap() { let expected = hashmap! {"foo".to_string() => 4}; let expected_json = json!({"foo": 4 }); - let code = "{'foo': 4}"; + let code = c_str!("{'foo': 4}"); test_de(code, &expected, &expected_json); } @@ -642,7 +699,7 @@ mod test { let expected = Foo::Variant; let expected_json = json!("Variant"); - let code = "'Variant'"; + let code = c_str!("'Variant'"); test_de(code, &expected, &expected_json); } @@ -655,7 +712,7 @@ mod test { let expected = Foo::Tuple(12, "cat".to_string()); let expected_json = json!({"Tuple": [12, "cat"]}); - let code = "{'Tuple': [12, 'cat']}"; + let code = c_str!("{'Tuple': [12, 'cat']}"); test_de(code, &expected, &expected_json); } @@ -668,7 +725,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!({"NewType": "cat" }); - let code = "{'NewType': 'cat'}"; + let code = c_str!("{'NewType': 'cat'}"); test_de(code, &expected, &expected_json); } @@ -684,7 +741,7 @@ mod test { bar: 25, }; let expected_json = json!({"Struct": {"foo": "cat", "bar": 25 }}); - let code = "{'Struct': {'foo': 'cat', 'bar': 25}}"; + let code = c_str!("{'Struct': {'foo': 'cat', 'bar': 25}}"); test_de(code, &expected, &expected_json); } #[test] @@ -697,7 +754,7 @@ mod test { let expected = Foo::Tuple(12.0, 'c'); let expected_json = json!([12.0, 'c']); - let code = "[12.0, 'c']"; + let code = c_str!("[12.0, 'c']"); test_de(code, &expected, &expected_json); } @@ -711,7 +768,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!("cat"); - let code = "'cat'"; + let code = c_str!("'cat'"); test_de(code, &expected, &expected_json); } @@ -728,7 +785,7 @@ mod test { bar: [2, 5, 3, 1], }; let expected_json = json!({"foo": ["a", "b", "c"], "bar": [2, 5, 3, 1]}); - let code = "{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"; + let code = c_str!("{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"); test_de(code, &expected, &expected_json); } @@ -761,39 +818,75 @@ mod test { }; let expected_json = json!({"name": "SomeFoo", "bar": { "value": 13, "variant": { "Tuple": [-1.5, 8]}}}); - let code = "{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"; + let code = + c_str!("{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"); test_de(code, &expected, &expected_json); } #[test] fn test_int_limits() { - Python::with_gil(|py| { - // serde_json::Value supports u64 and i64 as maxiumum sizes - let _: serde_json::Value = depythonize(&u8::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u8::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i8::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i8::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u16::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u16::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i16::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i16::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u32::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u32::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i32::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i32::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap(); - let _: serde_json::Value = depythonize(&i64::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: u128 = depythonize(&u128::MAX.into_py(py).into_bound(py)).unwrap(); - let _: i128 = depythonize(&u128::MIN.into_py(py).into_bound(py)).unwrap(); - - let _: i128 = depythonize(&i128::MAX.into_py(py).into_bound(py)).unwrap(); - let _: i128 = depythonize(&i128::MIN.into_py(py).into_bound(py)).unwrap(); + Python::attach(|py| { + // serde_json::Value supports u64 and i64 as maximum sizes + let _: serde_json::Value = depythonize(&u8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u8::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u16::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u32::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u64::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: u128 = depythonize(&u128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&u128::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: i128 = depythonize(&i128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&i128::MIN.into_pyobject(py).unwrap()).unwrap(); + }); + } + + #[test] + fn test_deserialize_bytes() { + Python::attach(|py| { + let obj = PyBytes::new(py, "hello".as_bytes()); + let actual: Vec = depythonize(&obj).unwrap(); + assert_eq!(actual, b"hello"); + }) + } + + #[test] + fn test_char() { + let expected = 'a'; + let expected_json = json!("a"); + let code = c_str!("'a'"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_unknown_type() { + Python::attach(|py| { + let obj = py + .import("decimal") + .unwrap() + .getattr("Decimal") + .unwrap() + .call0() + .unwrap(); + let err = depythonize::(&obj).unwrap_err(); + assert!(matches!( + *err.inner, + ErrorImpl::UnsupportedType(name) if name == "Decimal" + )); }); } } diff --git a/src/error.rs b/src/error.rs index 4aee7ea..b608106 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ use pyo3::PyErr; -use pyo3::{exceptions::*, DowncastError, DowncastIntoError}; +use pyo3::{exceptions::*, CastError, CastIntoError}; use serde::{de, ser}; +use std::convert::Infallible; use std::error; use std::fmt::{self, Debug, Display}; use std::result; @@ -72,7 +73,7 @@ pub enum ErrorImpl { Message(String), /// A Python type not supported by the deserializer UnsupportedType(String), - /// A `PyAny` object that failed to downcast to an expected Python type + /// A `PyAny` object that failed to cast to an expected Python type UnexpectedType(String), /// Dict keys should be strings to deserialize to struct fields DictKeyNotString, @@ -136,6 +137,13 @@ impl de::Error for PythonizeError { } } +/// Convert an exception raised in Python to a `PythonizeError` +impl From for PythonizeError { + fn from(other: Infallible) -> Self { + match other {} + } +} + /// Convert an exception raised in Python to a `PythonizeError` impl From for PythonizeError { fn from(other: PyErr) -> Self { @@ -145,18 +153,18 @@ impl From for PythonizeError { } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'a, 'py> From> for PythonizeError { - fn from(other: DowncastError<'a, 'py>) -> Self { +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'a, 'py> From> for PythonizeError { + fn from(other: CastError<'a, 'py>) -> Self { Self { inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), } } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'py> From> for PythonizeError { - fn from(other: DowncastIntoError<'py>) -> Self { +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'py> From> for PythonizeError { + fn from(other: CastIntoError<'py>) -> Self { Self { inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), } diff --git a/src/lib.rs b/src/lib.rs index f85a025..e625b6f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,50 +1,12 @@ -//! This crate converts Rust types which implement the [Serde] serialization -//! traits into Python objects using the [PyO3] library. -//! -//! Pythonize has two public APIs: `pythonize` and `depythonize`. -//! -//! [Serde]: https://github.com/serde-rs/serde -//! [PyO3]: https://github.com/PyO3/pyo3 -//! -//! # Examples -//! ``` -//! use serde::{Serialize, Deserialize}; -//! use pyo3::{types::PyAnyMethods, Python}; -//! use pythonize::{depythonize, pythonize}; -//! -//! #[derive(Debug, Serialize, Deserialize, PartialEq)] -//! struct Sample { -//! foo: String, -//! bar: Option -//! } -//! -//! Python::with_gil(|py| { -//! let sample = Sample { -//! foo: "Foo".to_string(), -//! bar: None -//! }; -//! -//! // Rust -> Python -//! let obj = pythonize(py, &sample).unwrap(); -//! -//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.repr().unwrap())); -//! -//! // Python -> Rust -//! let new_sample: Sample = depythonize(&obj).unwrap(); -//! -//! assert_eq!(new_sample, sample); -//! }); -//! -//! ``` +#![doc = include_str!("../README.md")] + mod de; mod error; mod ser; -#[allow(deprecated)] -pub use crate::de::depythonize_bound; pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, PythonizeDefault, PythonizeDictType, PythonizeListType, - PythonizeTypes, Pythonizer, + pythonize, pythonize_custom, PythonizeDefault, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index 282c011..c8e6dd1 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,56 +1,156 @@ use std::marker::PhantomData; -use pyo3::types::{PyAnyMethods, PyDict, PyList, PyMapping, PySequence, PyString, PyTuple}; -use pyo3::{Bound, IntoPy, PyAny, PyResult, Python, ToPyObject}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PySequence, PyString, PyTuple, + PyTupleMethods, +}; +use pyo3::{Bound, BoundObject, IntoPyObject, PyAny, PyResult, Python}; use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; /// Trait for types which can represent a Python mapping -pub trait PythonizeDictType { - /// Constructor - fn create_mapping(py: Python) -> PyResult>; +pub trait PythonizeMappingType { + /// Builder type for Python mappings + type Builder<'py>: 'py; + + /// Create a builder for a Python mapping + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult>; + + /// Adds the key-value item to the mapping being built + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()>; + + /// Build the Python mapping + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; +} + +/// Trait for types which can represent a Python mapping and have a name +pub trait PythonizeNamedMappingType { + /// Builder type for Python mappings with a name + type Builder<'py>: 'py; + + /// Create a builder for a Python mapping with a name + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult>; + + /// Adds the field to the named mapping being built + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()>; + + /// Build the Python mapping + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; } /// Trait for types which can represent a Python sequence pub trait PythonizeListType: Sized { /// Constructor - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator; } /// Custom types for serialization pub trait PythonizeTypes { /// Python map type (should be representable as python mapping) - type Map: PythonizeDictType; + type Map: PythonizeMappingType; + /// Python (struct-like) named map type (should be representable as python mapping) + type NamedMap: PythonizeNamedMappingType; /// Python sequence type (should be representable as python sequence) type List: PythonizeListType; } -impl PythonizeDictType for PyDict { - fn create_mapping(py: Python) -> PyResult> { - Ok(PyDict::new_bound(py).into_any().downcast_into().unwrap()) +impl PythonizeMappingType for PyDict { + type Builder<'py> = Bound<'py, Self>; + + fn builder<'py>(py: Python<'py>, _len: Option) -> PyResult> { + Ok(Self::new(py)) + } + + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + builder.set_item(key, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(builder.into_mapping()) + } +} + +/// Adapter type to use an unnamed mapping type, i.e. one that implements +/// [`PythonizeMappingType`], as a named mapping type, i.e. one that implements +/// [`PythonizeNamedMappingType`]. The adapter simply drops the provided name. +/// +/// This adapter is commonly applied to use the same unnamed mapping type for +/// both [`PythonizeTypes::Map`] and [`PythonizeTypes::NamedMap`] while only +/// implementing [`PythonizeMappingType`]. +pub struct PythonizeUnnamedMappingAdapter { + _unnamed: T, +} + +impl PythonizeNamedMappingType for PythonizeUnnamedMappingAdapter { + type Builder<'py> = T::Builder<'py>; + + fn builder<'py>( + py: Python<'py>, + len: usize, + _name: &'static str, + ) -> PyResult> { + T::builder(py, Some(len)) + } + + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + T::push_item(builder, name.into_any(), value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + T::finish(builder) } } impl PythonizeListType for PyList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { - Ok(PyList::new_bound(py, elements) - .into_any() - .downcast_into() - .unwrap()) + Ok(PyList::new(py, elements)?.into_sequence()) + } +} + +impl PythonizeListType for PyTuple { + fn create_sequence<'py, T, U>( + py: Python<'py>, + elements: impl IntoIterator, + ) -> PyResult> + where + T: IntoPyObject<'py>, + U: ExactSizeIterator, + { + Ok(PyTuple::new(py, elements)?.into_sequence()) } } @@ -58,6 +158,7 @@ pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyList; } @@ -116,31 +217,47 @@ pub struct PythonCollectionSerializer<'py, P> { #[doc(hidden)] pub struct PythonTupleVariantSerializer<'py, P> { + name: &'static str, variant: &'static str, inner: PythonCollectionSerializer<'py, P>, } #[doc(hidden)] pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { + name: &'static str, variant: &'static str, - inner: PythonDictSerializer<'py, P>, + inner: PythonStructDictSerializer<'py, P>, } #[doc(hidden)] -pub struct PythonDictSerializer<'py, P: PythonizeTypes> { +pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - dict: Bound<'py, PyMapping>, + builder: ::Builder<'py>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - map: Bound<'py, PyMapping>, + builder: ::Builder<'py>, key: Option>, _types: PhantomData

, } +impl<'py, P: PythonizeTypes> Pythonizer<'py, P> { + /// The default implementation for serialisation functions. + #[inline] + fn serialise_default(self, v: T) -> Result> + where + T: IntoPyObject<'py>, + >::Error: Into, + { + v.into_pyobject(self.py) + .map(|x| x.into_any().into_bound()) + .map_err(Into::into) + } +} + impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -149,51 +266,51 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { type SerializeTupleStruct = PythonCollectionSerializer<'py, P>; type SerializeTupleVariant = PythonTupleVariantSerializer<'py, P>; type SerializeMap = PythonMapSerializer<'py, P>; - type SerializeStruct = PythonDictSerializer<'py, P>; + type SerializeStruct = PythonStructDictSerializer<'py, P>; type SerializeStructVariant = PythonStructVariantSerializer<'py, P>; fn serialize_bool(self, v: bool) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i8(self, v: i8) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i16(self, v: i16) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i32(self, v: i32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_i64(self, v: i64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u8(self, v: u8) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u16(self, v: u16) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u32(self, v: u32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_u64(self, v: u64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_f32(self, v: f32) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_f64(self, v: f64) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_char(self, v: char) -> Result> { @@ -201,11 +318,11 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { } fn serialize_str(self, v: &str) -> Result> { - Ok(PyString::new_bound(self.py, v).into_any()) + Ok(PyString::new(self.py, v).into_any()) } fn serialize_bytes(self, v: &[u8]) -> Result> { - Ok(v.into_py(self.py).into_bound(self.py)) + self.serialise_default(v) } fn serialize_none(self) -> Result> { @@ -249,7 +366,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_newtype_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, value: &T, @@ -257,9 +374,13 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { where T: ?Sized + Serialize, { - let d = PyDict::new_bound(self.py); - d.set_item(variant, value.serialize(self)?)?; - Ok(d.into_any()) + let mut m = P::NamedMap::builder(self.py, 1, name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.py, variant), + value.serialize(self)?, + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } fn serialize_seq(self, len: Option) -> Result> { @@ -292,18 +413,22 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_tuple_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, len: usize, ) -> Result> { let inner = self.serialize_tuple(len)?; - Ok(PythonTupleVariantSerializer { variant, inner }) + Ok(PythonTupleVariantSerializer { + name, + variant, + inner, + }) } - fn serialize_map(self, _len: Option) -> Result> { + fn serialize_map(self, len: Option) -> Result> { Ok(PythonMapSerializer { - map: P::Map::create_mapping(self.py)?, + builder: P::Map::builder(self.py, len)?, key: None, py: self.py, _types: PhantomData, @@ -312,28 +437,29 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_struct( self, - _name: &'static str, - _len: usize, - ) -> Result> { - Ok(PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + name: &'static str, + len: usize, + ) -> Result> { + Ok(PythonStructDictSerializer { py: self.py, + builder: P::NamedMap::builder(self.py, len, name)?, _types: PhantomData, }) } fn serialize_struct_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, - _len: usize, + len: usize, ) -> Result> { Ok(PythonStructVariantSerializer { + name, variant, - inner: PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + inner: PythonStructDictSerializer { py: self.py, + builder: P::NamedMap::builder(self.py, len, variant)?, _types: PhantomData, }, }) @@ -354,7 +480,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'p fn end(self) -> Result> { let instance = P::List::create_sequence(self.py, self.items)?; - Ok(instance.to_object(self.py).into_bound(self.py)) + Ok(instance.into_pyobject(self.py)?.into_any()) } } @@ -370,7 +496,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer< } fn end(self) -> Result> { - Ok(PyTuple::new_bound(self.py, self.items).into_any()) + Ok(PyTuple::new(self.py, self.items)?.into_any()) } } @@ -402,9 +528,13 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe } fn end(self) -> Result> { - let d = PyDict::new_bound(self.inner.py); - d.set_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; - Ok(d.into_any()) + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.inner.py, self.variant), + ser::SerializeTuple::end(self.inner)?, + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } } @@ -424,7 +554,8 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { where T: ?Sized + Serialize, { - self.map.set_item( + P::Map::push_item( + &mut self.builder, self.key .take() .expect("serialize_value should always be called after serialize_key"), @@ -434,11 +565,11 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { } fn end(self) -> Result> { - Ok(self.map.into_any()) + Ok(P::Map::finish(self.builder)?.into_any()) } } -impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonDictSerializer<'py, P> { +impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { type Ok = Bound<'py, PyAny>; type Error = PythonizeError; @@ -446,13 +577,16 @@ impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonDictSerializer<'py, where T: ?Sized + Serialize, { - Ok(self - .dict - .set_item(key, pythonize_custom::(self.py, value)?)?) + P::NamedMap::push_field( + &mut self.builder, + PyString::new(self.py, key), + pythonize_custom::(self.py, value)?, + )?; + Ok(()) } fn end(self) -> Result> { - Ok(self.dict.into_any()) + Ok(P::NamedMap::finish(self.builder)?.into_any()) } } @@ -464,16 +598,23 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant where T: ?Sized + Serialize, { - self.inner - .dict - .set_item(key, pythonize_custom::(self.inner.py, value)?)?; + P::NamedMap::push_field( + &mut self.inner.builder, + PyString::new(self.inner.py, key), + pythonize_custom::(self.inner.py, value)?, + )?; Ok(()) } fn end(self) -> Result> { - let d = PyDict::new_bound(self.inner.py); - d.set_item(self.variant, self.inner.dict)?; - Ok(d.into_any()) + let v = P::NamedMap::finish(self.inner.builder)?; + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.inner.py, self.variant), + v.into_any(), + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } } @@ -481,6 +622,7 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant mod test { use super::pythonize; use maplit::hashmap; + use pyo3::ffi::c_str; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyBytes, PyDict}; @@ -490,14 +632,14 @@ mod test { where T: Serialize, { - Python::with_gil(|py| -> PyResult<()> { + Python::attach(|py| -> PyResult<()> { let obj = pythonize(py, &src)?; - let locals = PyDict::new_bound(py); + let locals = PyDict::new(py); locals.set_item("obj", obj)?; - py.run_bound( - "import json; result = json.dumps(obj, separators=(',', ':'))", + py.run( + c_str!("import json; result = json.dumps(obj, separators=(',', ':'))"), None, Some(&locals), )?; @@ -703,10 +845,10 @@ mod test { // serde treats &[u8] as a sequence of integers due to lack of specialization test_ser(b"foo", "[102,111,111]"); - Python::with_gil(|py| { + Python::attach(|py| { assert!(pythonize(py, serde_bytes::Bytes::new(b"foo")) .expect("bytes will always serialize successfully") - .eq(&PyBytes::new_bound(py, b"foo")) + .eq(&PyBytes::new(py, b"foo")) .expect("bytes will always compare successfully")); }); } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index 9027c74..27888d0 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -3,17 +3,19 @@ use std::collections::HashMap; use pyo3::{ exceptions::{PyIndexError, PyKeyError}, prelude::*, - types::{PyDict, PyList, PyMapping, PySequence}, + types::{PyDict, PyMapping, PySequence, PyTuple}, + IntoPyObjectExt, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, Pythonizer, + depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; use serde::Serialize; use serde_json::{json, Value}; #[pyclass(sequence)] struct CustomList { - items: Vec, + items: Vec>, } #[pymethods] @@ -22,7 +24,7 @@ impl CustomList { self.items.len() } - fn __getitem__(&self, idx: isize) -> PyResult { + fn __getitem__(&self, idx: isize) -> PyResult> { self.items .get(idx as usize) .cloned() @@ -31,12 +33,12 @@ impl CustomList { } impl PythonizeListType for CustomList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { let sequence = Bound::new( @@ -44,25 +46,25 @@ impl PythonizeListType for CustomList { CustomList { items: elements .into_iter() - .map(|item| item.to_object(py)) - .collect(), + .map(|item| item.into_py_any(py)) + .collect::>()?, }, - )? - .into_any(); + )?; - Ok(unsafe { sequence.downcast_into_unchecked() }) + Ok(unsafe { sequence.cast_into_unchecked() }) } } struct PythonizeCustomList; -impl PythonizeTypes for PythonizeCustomList { +impl<'py> PythonizeTypes for PythonizeCustomList { type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = CustomList; } #[test] fn test_custom_list() { - Python::with_gil(|py| { + Python::attach(|py| { PySequence::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!([1, 2, 3])).unwrap(); assert!(serialized.is_instance_of::()); @@ -74,7 +76,7 @@ fn test_custom_list() { #[pyclass(mapping)] struct CustomDict { - items: HashMap, + items: HashMap>, } #[pymethods] @@ -83,14 +85,14 @@ impl CustomDict { self.items.len() } - fn __getitem__(&self, key: String) -> PyResult { + fn __getitem__(&self, key: String) -> PyResult> { self.items .get(&key) .cloned() .ok_or_else(|| PyKeyError::new_err(key)) } - fn __setitem__(&mut self, key: String, value: PyObject) { + fn __setitem__(&mut self, key: String, value: Py) { self.items.insert(key, value); } @@ -98,33 +100,46 @@ impl CustomDict { self.items.keys().collect() } - fn values(&self) -> Vec { + fn values(&self) -> Vec> { self.items.values().cloned().collect() } } -impl PythonizeDictType for CustomDict { - fn create_mapping(py: Python) -> PyResult> { - let mapping = Bound::new( +impl PythonizeMappingType for CustomDict { + type Builder<'py> = Bound<'py, CustomDict>; + + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult> { + Bound::new( py, CustomDict { - items: HashMap::new(), + items: HashMap::with_capacity(len.unwrap_or(0)), }, - )? - .into_any(); - Ok(unsafe { mapping.downcast_into_unchecked() }) + ) + } + + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + unsafe { builder.cast_unchecked::() }.set_item(key, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(unsafe { builder.cast_into_unchecked() }) } } struct PythonizeCustomDict; -impl PythonizeTypes for PythonizeCustomDict { +impl<'py> PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; - type List = PyList; + type NamedMap = PythonizeUnnamedMappingAdapter; + type List = PyTuple; } #[test] fn test_custom_dict() { - Python::with_gil(|py| { + Python::attach(|py| { PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) @@ -136,10 +151,23 @@ fn test_custom_dict() { }) } +#[test] +fn test_tuple() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &json!([1, 2, 3, 4])).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!([1, 2, 3, 4])); + }) +} + #[test] fn test_pythonizer_can_be_created() { // https://github.com/davidhewitt/pythonize/pull/56 - Python::with_gil(|py| { + Python::attach(|py| { let sample = json!({ "hello": 1, "world": 2 }); assert!(sample .serialize(Pythonizer::new(py)) @@ -152,3 +180,106 @@ fn test_pythonizer_can_be_created() { .is_instance_of::()); }) } + +#[pyclass(mapping)] +struct NamedCustomDict { + name: String, + items: HashMap>, +} + +#[pymethods] +impl NamedCustomDict { + fn __len__(&self) -> usize { + self.items.len() + } + + fn __getitem__(&self, key: String) -> PyResult> { + self.items + .get(&key) + .cloned() + .ok_or_else(|| PyKeyError::new_err(key)) + } + + fn __setitem__(&mut self, key: String, value: Py) { + self.items.insert(key, value); + } + + fn keys(&self) -> Vec<&String> { + self.items.keys().collect() + } + + fn values(&self) -> Vec> { + self.items.values().cloned().collect() + } +} + +impl PythonizeNamedMappingType for NamedCustomDict { + type Builder<'py> = Bound<'py, NamedCustomDict>; + + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult> { + Bound::new( + py, + NamedCustomDict { + name: String::from(name), + items: HashMap::with_capacity(len), + }, + ) + } + + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, pyo3::types::PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + unsafe { builder.cast_unchecked::() }.set_item(name, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(unsafe { builder.cast_into_unchecked() }) + } +} + +struct PythonizeNamedCustomDict; +impl<'py> PythonizeTypes for PythonizeNamedCustomDict { + type Map = CustomDict; + type NamedMap = NamedCustomDict; + type List = PyTuple; +} + +#[derive(Serialize)] +struct Struct { + hello: u8, + world: i8, +} + +#[test] +fn test_custom_unnamed_dict() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} + +#[test] +fn test_custom_named_dict() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }) + .unwrap(); + let named: Bound = serialized.extract().unwrap(); + assert_eq!(named.borrow().name, "Struct"); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs index acb34d7..82fd8bb 100644 --- a/tests/test_with_serde_path_to_err.rs +++ b/tests/test_with_serde_path_to_err.rs @@ -4,7 +4,7 @@ use pyo3::{ prelude::*, types::{PyDict, PyList}, }; -use pythonize::PythonizeTypes; +use pythonize::{PythonizeTypes, PythonizeUnnamedMappingAdapter}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -13,8 +13,9 @@ struct Root { root_map: BTreeMap>, } -impl PythonizeTypes for Root { +impl<'py, T> PythonizeTypes for Root { type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyList; } @@ -39,15 +40,15 @@ impl Serialize for CannotSerialize { #[test] fn test_de_valid() { - Python::with_gil(|py| { - let pyroot = PyDict::new_bound(py); + Python::attach(|py| { + let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new_bound(py); - let nested_0 = PyDict::new_bound(py); + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new_bound(py); + let nested_1 = PyDict::new(py); nested_1.set_item("nested_key", "nested_value_1").unwrap(); nested.set_item("nested_1", nested_1).unwrap(); @@ -81,15 +82,15 @@ fn test_de_valid() { #[test] fn test_de_invalid() { - Python::with_gil(|py| { - let pyroot = PyDict::new_bound(py); + Python::attach(|py| { + let pyroot = PyDict::new(py); pyroot.set_item("root_key", "root_value").unwrap(); - let nested = PyDict::new_bound(py); - let nested_0 = PyDict::new_bound(py); + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); nested_0.set_item("nested_key", "nested_value_0").unwrap(); nested.set_item("nested_0", nested_0).unwrap(); - let nested_1 = PyDict::new_bound(py); + let nested_1 = PyDict::new(py); nested_1.set_item("nested_key", 1).unwrap(); nested.set_item("nested_1", nested_1).unwrap(); @@ -99,13 +100,16 @@ fn test_de_invalid() { let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); - assert_eq!(err.to_string(), "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be converted to 'PyString'"); + assert_eq!( + err.to_string(), + "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be cast as 'str'" + ); }) } #[test] fn test_ser_valid() { - Python::with_gil(|py| { + Python::attach(|py| { let root = Root { root_key: String::from("root_value"), root_map: BTreeMap::from([ @@ -127,7 +131,7 @@ fn test_ser_valid() { let ser = pythonize::Pythonizer::>::from(py); let pyroot: Bound<'_, PyAny> = serde_path_to_error::serialize(&root, ser).unwrap(); - let pyroot = pyroot.downcast::().unwrap(); + let pyroot = pyroot.cast::().unwrap(); assert_eq!(pyroot.len(), 2); let root_value: String = pyroot @@ -142,7 +146,7 @@ fn test_ser_valid() { .get_item("root_map") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(root_map.len(), 2); @@ -150,7 +154,7 @@ fn test_ser_valid() { .get_item("nested_0") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(nested_0.len(), 1); let nested_key_0: String = nested_0 @@ -165,7 +169,7 @@ fn test_ser_valid() { .get_item("nested_1") .unwrap() .unwrap() - .downcast_into::() + .cast_into::() .unwrap(); assert_eq!(nested_1.len(), 1); let nested_key_1: String = nested_1 @@ -180,7 +184,7 @@ fn test_ser_valid() { #[test] fn test_ser_invalid() { - Python::with_gil(|py| { + Python::attach(|py| { let root = Root { root_key: String::from("root_value"), root_map: BTreeMap::from([