diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c1ce3c3..8821fa7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,8 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7133743..3d69440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,21 +3,32 @@ name: CI on: push: branches: - - master + - main pull_request: env: CARGO_TERM_COLOR: always jobs: + resolve: + runs-on: ubuntu-latest + outputs: + MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: resolve MSRV + id: resolve-msrv + run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT + fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable - profile: minimal components: rustfmt - name: Check rust formatting (rustfmt) run: cargo fmt --all -- --check @@ -25,48 +36,63 @@ jobs: clippy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable - profile: minimal components: clippy - run: cargo clippy --all build: - needs: [fmt] # don't wait for clippy as fails rarely and takes longer - name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} ${{ matrix.msrv }} - runs-on: ${{ matrix.platform.os }} + needs: [resolve, fmt] # don't wait for clippy as fails rarely and takes longer + name: python${{ matrix.python-version }} ${{ matrix.os }} rust-${{ matrix.rust}} + runs-on: ${{ matrix.os }} strategy: - fail-fast: false # If one platform fails, allow the rest to keep testing. + fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] - platform: [ - { os: "macOS-latest", python-architecture: "x64", rust-target: "x86_64-apple-darwin" }, - { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" }, - { os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" }, - { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" }, - ] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] + os: ["macos-latest", "ubuntu-latest", "windows-latest"] + rust: [stable] + include: + - python-version: "3.14" + os: "ubuntu-latest" + rust: ${{ needs.resolve.outputs.MSRV }} + - python-version: "3.14" + os: "macos-15-intel" + rust: "stable" + - python-version: "3.14" + os: "ubuntu-24.04-arm" + rust: "stable" + - python-version: "3.14" + os: "windows-11-arm" + rust: "stable" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.platform.python-architecture }} - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - toolchain: stable - target: ${{ matrix.platform.rust-target }} - profile: minimal - default: true + toolchain: ${{ matrix.rust }} - - name: Build without default features - run: cargo test --no-default-features --verbose --target ${{ matrix.platform.rust-target }} + - uses: Swatinem/rust-cache@v2 + continue-on-error: true + + - if: ${{ matrix.rust == needs.resolve.outputs.MSRV }} + name: Set dependencies on MSRV + run: cargo +stable update + env: + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback + + - name: Test + run: cargo test --verbose + + - name: Test (abi3) + run: cargo test --verbose --features pyo3/abi3-py37 env: RUST_BACKTRACE: 1 @@ -75,30 +101,18 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: coverage-cargo-${{ hashFiles('**/Cargo.toml') }} + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 continue-on-error: true - - name: install cargo-llvm-cov - run: | - wget https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz -qO- | tar -xzvf - - mv cargo-llvm-cov ~/.cargo/bin - env: - CARGO_LLVM_COV_VERSION: 0.1.9 - - uses: actions-rs/toolchain@v1 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - uses: dtolnay/rust-toolchain@stable with: - toolchain: nightly - override: true - profile: minimal components: llvm-tools-preview - run: | cargo llvm-cov clean - cargo llvm-cov --lcov --output-path coverage.lcov - - uses: codecov/codecov-action@v2 + cargo llvm-cov --codecov --output-path codecov.json + - uses: codecov/codecov-action@v4 with: - file: coverage.lcov + file: codecov.json + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f538d3d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: Release Rust Crate + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: The version to build + +jobs: + release: + permissions: + id-token: write + + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@v5 + with: + # The tag to build or the tag received by the tag event + ref: ${{ github.event.inputs.version || github.ref }} + persist-credentials: false + + - uses: rust-lang/crates-io-auth-action@v1 + id: auth + + - name: Publish to crates.io + run: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d58d8..f267740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,87 @@ +## 0.27.0 - 2025-11-07 +- Update to PyO3 0.27 + +## 0.26.0 - 2025-08-30 + +### Packaging +- Bump MSRV to 1.74 +- Update to PyO3 0.26 + +### Changed +- `PythonizeTypes`, `PythonizeMappingType` and `PythonizeNamedMappingType` no longer have a lifetime on the trait, instead the `Builder` type is a GAT. + +## 0.25.0 - 2025-05-23 + +### Packaging +- Update to PyO3 0.25 + +## 0.24.0 - 2025-03-26 + +### Packaging +- Update to PyO3 0.24 + +## Removed +- Remove deprecated `depythonize_bound()` + +## 0.23.0 - 2024-11-22 + +### Packaging +- Update to PyO3 0.23 + +## 0.22.0 - 2024-08-10 + +### Packaging +- Bump MSRV to 1.63 +- Update to PyO3 0.22 + +### Added +- Support `u128` / `i128` integers. +- Implement `PythonizeListType` for `PyTuple` +- Support deserializing enums from any `PyMapping` instead of just `PyDict` +- Support serializing struct-like types to named mappings using `PythonizeTypes::NamedMap` + +### Changed +- `pythonize()` now returns `Bound<'py, PyAny>` instead of `Py` +- `depythonize()` now take `&'a Bound` and is no longer deprecated +- `depythonize_bound()` is now deprecated +- `Depythonizer::from_object()` now takes `&'a Bound` and is no longer deprecated +- `Depythonizer` now contains `&'a Bound` and so has an extra lifetime `'a` + +### Removed +- Remove support for PyO3's `gil-refs` feature + +### Fixed +- Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` +- Fix deserializing `set` and `frozenset` into Rust homogeneous containers + +## 0.21.1 - 2024-04-02 + +- Fix compile error when using PyO3 `abi3` feature targeting a minimum version below 3.10 + +## 0.21.0 - 2024-04-01 + +- Bump edition to 2021 +- Bump MSRV to 1.56 +- Update to PyO3 0.21 +- Export `PythonizeDefault` + +## 0.20.0 - 2023-10-15 + +- Update to PyO3 0.20 + +## 0.19.0 - 2023-06-11 + +- Update to PyO3 0.19 + +## 0.18.0 - 2023-01-22 + +- Add LICENSE file to the crate +- Update to PyO3 0.18 + +## 0.17.0 - 2022-08-24 + +- Update to PyO3 0.17 + ## 0.16.0 - 2022-03-06 - Update to PyO3 0.16 diff --git a/Cargo.toml b/Cargo.toml index 3ff2974..4a714a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "pythonize" -version = "0.16.0" +version = "0.27.0" authors = ["David Hewitt <1939362+davidhewitt@users.noreply.github.com>"] -edition = "2018" +edition = "2021" +rust-version = "1.74" license = "MIT" description = "Serde Serializer & Deserializer from Rust <--> Python, backed by PyO3." homepage = "https://github.com/davidhewitt/pythonize" @@ -12,10 +13,12 @@ documentation = "https://docs.rs/crate/pythonize/" [dependencies] serde = { version = "1.0", default-features = false, features = ["std"] } -pyo3 = { version = "0.16.3", default-features = false } +pyo3 = { version = "0.27", default-features = false } [dev-dependencies] serde = { version = "1.0", default-features = false, features = ["derive"] } -pyo3 = { version = "0.16.3", default-features = false, features = ["auto-initialize", "macros", "pyproto"] } +pyo3 = { version = "0.27", default-features = false, features = ["auto-initialize", "macros", "py-clone"] } serde_json = "1.0" +serde_bytes = "0.11" maplit = "1.0.2" +serde_path_to_error = "0.1.15" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ace026e --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2022-present David Hewitt and Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 78170fb..2667523 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ that which is produced directly by `pythonize`. This crate converts Rust types which implement the [Serde] serialization traits into Python objects using the [PyO3] library. -Pythonize has two public APIs: `pythonize` and `depythonize`. +Pythonize has two main public APIs: `pythonize` and `depythonize`. + + [Serde]: https://github.com/serde-rs/serde [PyO3]: https://github.com/PyO3/pyo3 @@ -19,7 +21,7 @@ Pythonize has two public APIs: `pythonize` and `depythonize`. ```rust use serde::{Serialize, Deserialize}; -use pyo3::Python; +use pyo3::prelude::*; use pythonize::{depythonize, pythonize}; #[derive(Debug, Serialize, Deserialize, PartialEq)] @@ -28,21 +30,20 @@ struct Sample { bar: Option } -let gil = Python::acquire_gil(); -let py = gil.python(); - let sample = Sample { foo: "Foo".to_string(), bar: None }; -// Rust -> Python -let obj = pythonize(py, &sample).unwrap(); +Python::attach(|py| { + // Rust -> Python + let obj = pythonize(py, &sample).unwrap(); -assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); + assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.repr().unwrap())); -// Python -> Rust -let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); + // Python -> Rust + let new_sample: Sample = depythonize(&obj).unwrap(); -assert_eq!(new_sample, sample); + assert_eq!(new_sample, sample); +}) ``` diff --git a/src/de.rs b/src/de.rs index 1f4253d..a30dbca 100644 --- a/src/de.rs +++ b/src/de.rs @@ -1,29 +1,30 @@ -use pyo3::types::*; +use pyo3::{types::*, Bound}; use serde::de::{self, IntoDeserializer}; use serde::Deserialize; -use crate::error::{PythonizeError, Result}; +use crate::error::{ErrorImpl, PythonizeError, Result}; /// Attempt to convert a Python object to an instance of `T` -pub fn depythonize<'de, T>(obj: &'de PyAny) -> Result +pub fn depythonize<'a, 'py, T>(obj: &'a Bound<'py, PyAny>) -> Result where - T: Deserialize<'de>, + T: Deserialize<'a>, { - let mut depythonizer = Depythonizer::from_object(obj); - T::deserialize(&mut depythonizer) + T::deserialize(&mut Depythonizer::from_object(obj)) } -pub struct Depythonizer<'de> { - input: &'de PyAny, +/// A structure that deserializes Python objects into Rust values +pub struct Depythonizer<'a, 'py> { + input: &'a Bound<'py, PyAny>, } -impl<'de> Depythonizer<'de> { - pub fn from_object(input: &'de PyAny) -> Self { +impl<'a, 'py> Depythonizer<'a, 'py> { + /// Create a deserializer from a Python object + pub fn from_object(input: &'a Bound<'py, PyAny>) -> Self { Depythonizer { input } } - fn sequence_access(&self, expected_len: Option) -> Result> { - let seq: &PySequence = self.input.downcast()?; + fn sequence_access(&self, expected_len: Option) -> Result> { + let seq = self.input.cast::()?; let len = self.input.len()?; match expected_len { @@ -34,8 +35,53 @@ impl<'de> Depythonizer<'de> { } } - fn dict_access(&self) -> Result> { - PyMappingAccess::new(self.input.downcast()?) + fn set_access(&self) -> Result> { + match self.input.cast::() { + Ok(set) => Ok(PySetAsSequence::from_set(set)), + Err(e) => { + if let Ok(f) = self.input.cast::() { + Ok(PySetAsSequence::from_frozenset(f)) + } else { + Err(e.into()) + } + } + } + } + + fn dict_access(&self) -> Result> { + PyMappingAccess::new(self.input.cast()?) + } + + fn deserialize_any_int<'de, V>(&self, int: &Bound<'_, PyInt>, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + if let Ok(x) = int.extract::() { + if let Ok(x) = u8::try_from(x) { + visitor.visit_u8(x) + } else if let Ok(x) = u16::try_from(x) { + visitor.visit_u16(x) + } else if let Ok(x) = u32::try_from(x) { + visitor.visit_u32(x) + } else if let Ok(x) = u64::try_from(x) { + visitor.visit_u64(x) + } else { + visitor.visit_u128(x) + } + } else { + let x: i128 = int.extract()?; + if let Ok(x) = i8::try_from(x) { + visitor.visit_i8(x) + } else if let Ok(x) = i16::try_from(x) { + visitor.visit_i16(x) + } else if let Ok(x) = i32::try_from(x) { + visitor.visit_i32(x) + } else if let Ok(x) = i64::try_from(x) { + visitor.visit_i64(x) + } else { + visitor.visit_i128(x) + } + } } } @@ -50,7 +96,7 @@ macro_rules! deserialize_type { }; } -impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { +impl<'de> de::Deserializer<'de> for &'_ mut Depythonizer<'_, '_> { type Error = PythonizeError; fn deserialize_any(self, visitor: V) -> Result @@ -59,39 +105,37 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { { let obj = self.input; + // First check for cases which are cheap to check due to pointer + // comparison or bitflag checks if obj.is_none() { self.deserialize_unit(visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if obj.is_instance_of::()? || obj.is_instance_of::()? { - self.deserialize_bytes(visitor) - } else if obj.is_instance_of::()? { - self.deserialize_map(visitor) - } else if obj.is_instance_of::()? { - self.deserialize_f64(visitor) - } else if obj.is_instance_of::()? { - self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { - self.deserialize_i64(visitor) - } else if obj.is_instance_of::()? { - self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { - self.deserialize_i64(visitor) - } else if obj.is_instance_of::()? { - self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { - self.deserialize_str(visitor) - } else if obj.is_instance_of::()? { + } else if let Ok(x) = obj.cast::() { + self.deserialize_any_int(x, visitor) + } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.is_instance_of::()? { + } else if obj.is_instance_of::() { + self.deserialize_map(visitor) + } else if obj.is_instance_of::() { self.deserialize_str(visitor) - } else if let Ok(_) = obj.downcast::() { + } + // Continue with cases which are slower to check because they go + // through `isinstance` machinery + else if obj.is_instance_of::() || obj.is_instance_of::() { + self.deserialize_bytes(visitor) + } else if obj.is_instance_of::() { + self.deserialize_f64(visitor) + } else if obj.is_instance_of::() || obj.is_instance_of::() { + self.deserialize_seq(visitor) + } else if obj.cast::().is_ok() { self.deserialize_tuple(obj.len()?, visitor) - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { self.deserialize_map(visitor) } else { - Err(PythonizeError::unsupported_type( - obj.get_type().name().unwrap_or(""), + Err(obj.get_type().qualname().map_or_else( + |_| PythonizeError::unsupported_type("unknown"), + PythonizeError::unsupported_type, )) } } @@ -100,14 +144,14 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - visitor.visit_bool(self.input.is_true()?) + visitor.visit_bool(self.input.is_truthy()?) } fn deserialize_char(self, visitor: V) -> Result where V: de::Visitor<'de>, { - let s = self.input.cast_as::()?.to_str()?; + let s = self.input.cast::()?.to_cow()?; if s.len() != 1 { return Err(PythonizeError::invalid_length_char()); } @@ -118,10 +162,12 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { deserialize_type!(deserialize_i16 => visit_i16); deserialize_type!(deserialize_i32 => visit_i32); deserialize_type!(deserialize_i64 => visit_i64); + deserialize_type!(deserialize_i128 => visit_i128); deserialize_type!(deserialize_u8 => visit_u8); deserialize_type!(deserialize_u16 => visit_u16); deserialize_type!(deserialize_u32 => visit_u32); deserialize_type!(deserialize_u64 => visit_u64); + deserialize_type!(deserialize_u128 => visit_u128); deserialize_type!(deserialize_f32 => visit_f32); deserialize_type!(deserialize_f64 => visit_f64); @@ -129,8 +175,8 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s: &PyString = self.input.cast_as()?; - visitor.visit_str(s.to_str()?) + let s = self.input.cast::()?; + visitor.visit_str(&s.to_cow()?) } fn deserialize_string(self, visitor: V) -> Result @@ -144,8 +190,7 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let obj = self.input; - let b: &PyBytes = obj.cast_as()?; + let b = self.input.cast::()?; visitor.visit_bytes(b.as_bytes()) } @@ -196,7 +241,18 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - visitor.visit_seq(self.sequence_access(None)?) + match self.sequence_access(None) { + Ok(seq) => visitor.visit_seq(seq), + Err(e) => { + // we allow sets to be deserialized as sequences, so try that + if matches!(*e.inner, ErrorImpl::UnexpectedType(_)) { + if let Ok(set) = self.set_access() { + return visitor.visit_seq(set); + } + } + Err(e) + } + } } fn deserialize_tuple(self, len: usize, visitor: V) -> Result @@ -246,24 +302,21 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let item = self.input; - if item.is_instance_of::()? { - // Get the enum variant from the dict key - let d: &PyDict = item.cast_as().unwrap(); - if d.len() != 1 { + let item = &self.input; + if let Ok(s) = item.cast::() { + visitor.visit_enum(s.to_cow()?.into_deserializer()) + } else if let Ok(m) = item.cast::() { + // Get the enum variant from the mapping key + if m.len()? != 1 { return Err(PythonizeError::invalid_length_enum()); } - let variant: &PyString = d - .keys() + let variant: Bound = m + .keys()? .get_item(0)? - .cast_as() + .cast_into::() .map_err(|_| PythonizeError::dict_key_not_string())?; - let value = d.get_item(variant).unwrap(); - let mut de = Depythonizer::from_object(value); - visitor.visit_enum(PyEnumAccess::new(&mut de, variant)) - } else if item.is_instance_of::()? { - let s: &PyString = self.input.cast_as()?; - visitor.visit_enum(s.to_str()?.into_deserializer()) + let value = m.get_item(&variant)?; + visitor.visit_enum(PyEnumAccess::new(&value, variant)) } else { Err(PythonizeError::invalid_enum_type()) } @@ -273,11 +326,11 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { where V: de::Visitor<'de>, { - let s: &PyString = self + let s = self .input - .cast_as() + .cast::() .map_err(|_| PythonizeError::dict_key_not_string())?; - visitor.visit_str(s.to_str()?) + visitor.visit_str(&s.to_cow()?) } fn deserialize_ignored_any(self, visitor: V) -> Result @@ -288,19 +341,19 @@ impl<'a, 'de> de::Deserializer<'de> for &'a mut Depythonizer<'de> { } } -struct PySequenceAccess<'a> { - seq: &'a PySequence, +struct PySequenceAccess<'a, 'py> { + seq: &'a Bound<'py, PySequence>, index: usize, len: usize, } -impl<'a> PySequenceAccess<'a> { - fn new(seq: &'a PySequence, len: usize) -> Self { +impl<'a, 'py> PySequenceAccess<'a, 'py> { + fn new(seq: &'a Bound<'py, PySequence>, len: usize) -> Self { Self { seq, index: 0, len } } } -impl<'de> de::SeqAccess<'de> for PySequenceAccess<'de> { +impl<'de> de::SeqAccess<'de> for PySequenceAccess<'_, '_> { type Error = PythonizeError; fn next_element_seed(&mut self, seed: T) -> Result> @@ -308,25 +361,60 @@ impl<'de> de::SeqAccess<'de> for PySequenceAccess<'de> { T: de::DeserializeSeed<'de>, { if self.index < self.len { - let mut item_de = Depythonizer::from_object(self.seq.get_item(self.index)?); + let item = self.seq.get_item(self.index)?; self.index += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } } } -struct PyMappingAccess<'de> { - keys: &'de PySequence, - values: &'de PySequence, +struct PySetAsSequence<'py> { + iter: Bound<'py, PyIterator>, +} + +impl<'py> PySetAsSequence<'py> { + fn from_set(set: &Bound<'py, PySet>) -> Self { + Self { + iter: PyIterator::from_object(set).expect("set is always iterable"), + } + } + + fn from_frozenset(set: &Bound<'py, PyFrozenSet>) -> Self { + Self { + iter: PyIterator::from_object(set).expect("frozenset is always iterable"), + } + } +} + +impl<'de> de::SeqAccess<'de> for PySetAsSequence<'_> { + type Error = PythonizeError; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: de::DeserializeSeed<'de>, + { + match self.iter.next() { + Some(item) => seed + .deserialize(&mut Depythonizer::from_object(&item?)) + .map(Some), + None => Ok(None), + } + } +} + +struct PyMappingAccess<'py> { + keys: Bound<'py, PyList>, + values: Bound<'py, PyList>, key_idx: usize, val_idx: usize, len: usize, } -impl<'de> PyMappingAccess<'de> { - fn new(map: &'de PyMapping) -> Result { +impl<'py> PyMappingAccess<'py> { + fn new(map: &Bound<'py, PyMapping>) -> Result { let keys = map.keys()?; let values = map.values()?; let len = map.len()?; @@ -340,7 +428,7 @@ impl<'de> PyMappingAccess<'de> { } } -impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { +impl<'de> de::MapAccess<'de> for PyMappingAccess<'_> { type Error = PythonizeError; fn next_key_seed(&mut self, seed: K) -> Result> @@ -348,9 +436,10 @@ impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { K: de::DeserializeSeed<'de>, { if self.key_idx < self.len { - let mut item_de = Depythonizer::from_object(self.keys.get_item(self.key_idx)?); + let item = self.keys.get_item(self.key_idx)?; self.key_idx += 1; - seed.deserialize(&mut item_de).map(Some) + seed.deserialize(&mut Depythonizer::from_object(&item)) + .map(Some) } else { Ok(None) } @@ -360,24 +449,27 @@ impl<'de> de::MapAccess<'de> for PyMappingAccess<'de> { where V: de::DeserializeSeed<'de>, { - let mut item_de = Depythonizer::from_object(self.values.get_item(self.val_idx)?); + let item = self.values.get_item(self.val_idx)?; self.val_idx += 1; - seed.deserialize(&mut item_de) + seed.deserialize(&mut Depythonizer::from_object(&item)) } } -struct PyEnumAccess<'a, 'de> { - de: &'a mut Depythonizer<'de>, - variant: &'de PyString, +struct PyEnumAccess<'a, 'py> { + de: Depythonizer<'a, 'py>, + variant: Bound<'py, PyString>, } -impl<'a, 'de> PyEnumAccess<'a, 'de> { - fn new(de: &'a mut Depythonizer<'de>, variant: &'de PyString) -> Self { - Self { de, variant } +impl<'a, 'py> PyEnumAccess<'a, 'py> { + fn new(obj: &'a Bound<'py, PyAny>, variant: Bound<'py, PyString>) -> Self { + Self { + de: Depythonizer::from_object(obj), + variant, + } } } -impl<'a, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'de> { +impl<'de> de::EnumAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; type Variant = Self; @@ -385,14 +477,14 @@ impl<'a, 'de> de::EnumAccess<'de> for PyEnumAccess<'a, 'de> { where V: de::DeserializeSeed<'de>, { - let de: de::value::StrDeserializer<'_, PythonizeError> = - self.variant.to_str()?.into_deserializer(); + let cow = self.variant.to_cow()?; + let de: de::value::StrDeserializer<'_, PythonizeError> = cow.as_ref().into_deserializer(); let val = seed.deserialize(de)?; Ok((val, self)) } } -impl<'a, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'de> { +impl<'de> de::VariantAccess<'de> for PyEnumAccess<'_, '_> { type Error = PythonizeError; fn unit_variant(self) -> Result<()> { @@ -403,7 +495,7 @@ impl<'a, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'de> { where T: de::DeserializeSeed<'de>, { - seed.deserialize(self.de) + seed.deserialize(&mut { self.de }) } fn tuple_variant(self, len: usize, visitor: V) -> Result @@ -423,26 +515,27 @@ impl<'a, 'de> de::VariantAccess<'de> for PyEnumAccess<'a, 'de> { #[cfg(test)] mod test { + use std::ffi::CStr; + use super::*; use crate::error::ErrorImpl; use maplit::hashmap; - use pyo3::Python; + use pyo3::ffi::c_str; + use pyo3::{IntoPyObject, Python}; use serde_json::{json, Value as JsonValue}; - fn test_de(code: &str, expected: &T, expected_json: &JsonValue) + fn test_de(code: &CStr, expected: &T, expected_json: &JsonValue) where T: de::DeserializeOwned + PartialEq + std::fmt::Debug, { - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - let actual: T = depythonize(obj).unwrap(); - assert_eq!(&actual, expected); - let actual_json: JsonValue = depythonize(obj).unwrap(); - assert_eq!(&actual_json, expected_json); + Python::attach(|py| { + let obj = py.eval(code, None, None).unwrap(); + let actual: T = depythonize(&obj).unwrap(); + assert_eq!(&actual, expected); + + let actual_json: JsonValue = depythonize(&obj).unwrap(); + assert_eq!(&actual_json, expected_json); + }); } #[test] @@ -452,7 +545,7 @@ mod test { let expected = Empty; let expected_json = json!(null); - let code = "None"; + let code = c_str!("None"); test_de(code, &expected, &expected_json); } @@ -463,19 +556,22 @@ mod test { foo: String, bar: usize, baz: f32, + qux: bool, } let expected = Struct { foo: "Foo".to_string(), bar: 8usize, baz: 45.23, + qux: true, }; let expected_json = json!({ "foo": "Foo", "bar": 8, - "baz": 45.23 + "baz": 45.23, + "qux": true }); - let code = "{'foo': 'Foo', 'bar': 8, 'baz': 45.23}"; + let code = c_str!("{'foo': 'Foo', 'bar': 8, 'baz': 45.23, 'qux': True}"); test_de(code, &expected, &expected_json); } @@ -487,18 +583,16 @@ mod test { bar: usize, } - let code = "{'foo': 'Foo'}"; + let code = c_str!("{'foo': 'Foo'}"); - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - assert!(matches!( - *depythonize::(obj).unwrap_err().inner, - ErrorImpl::Message(msg) if msg == "missing field `bar`" - )); + Python::attach(|py| { + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); + assert!(matches!( + *depythonize::(&obj).unwrap_err().inner, + ErrorImpl::Message(msg) if msg == "missing field `bar`" + )); + }) } #[test] @@ -508,7 +602,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "('cat', -10.05)"; + let code = c_str!("('cat', -10.05)"); test_de(code, &expected, &expected_json); } @@ -517,18 +611,16 @@ mod test { #[derive(Debug, Deserialize, PartialEq)] struct TupleStruct(String, f64); - let code = "('cat', -10.05, 'foo')"; + let code = c_str!("('cat', -10.05, 'foo')"); - let gil = Python::acquire_gil(); - let py = gil.python(); - let locals = PyDict::new(py); - py.run(&format!("obj = {}", code), None, Some(locals)) - .unwrap(); - let obj = locals.get_item("obj").unwrap(); - assert!(matches!( - *depythonize::(obj).unwrap_err().inner, - ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 - )); + Python::attach(|py| { + let locals = PyDict::new(py); + let obj = py.eval(code, None, Some(&locals)).unwrap(); + assert!(matches!( + *depythonize::(&obj).unwrap_err().inner, + ErrorImpl::IncorrectSequenceLength { expected, got } if expected == 2 && got == 3 + )); + }) } #[test] @@ -538,7 +630,7 @@ mod test { let expected = TupleStruct("cat".to_string(), -10.05); let expected_json = json!(["cat", -10.05]); - let code = "['cat', -10.05]"; + let code = c_str!("['cat', -10.05]"); test_de(code, &expected, &expected_json); } @@ -546,7 +638,7 @@ mod test { fn test_tuple() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "('foo', 5)"; + let code = c_str!("('foo', 5)"); test_de(code, &expected, &expected_json); } @@ -554,7 +646,23 @@ mod test { fn test_tuple_from_pylist() { let expected = ("foo".to_string(), 5); let expected_json = json!(["foo", 5]); - let code = "['foo', 5]"; + let code = c_str!("['foo', 5]"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_vec_from_pyset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = c_str!("{'foo'}"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_vec_from_pyfrozenset() { + let expected = vec!["foo".to_string()]; + let expected_json = json!(["foo"]); + let code = c_str!("frozenset({'foo'})"); test_de(code, &expected, &expected_json); } @@ -562,7 +670,7 @@ mod test { fn test_vec() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "[3, 2, 1]"; + let code = c_str!("[3, 2, 1]"); test_de(code, &expected, &expected_json); } @@ -570,7 +678,7 @@ mod test { fn test_vec_from_tuple() { let expected = vec![3, 2, 1]; let expected_json = json!([3, 2, 1]); - let code = "(3, 2, 1)"; + let code = c_str!("(3, 2, 1)"); test_de(code, &expected, &expected_json); } @@ -578,7 +686,7 @@ mod test { fn test_hashmap() { let expected = hashmap! {"foo".to_string() => 4}; let expected_json = json!({"foo": 4 }); - let code = "{'foo': 4}"; + let code = c_str!("{'foo': 4}"); test_de(code, &expected, &expected_json); } @@ -591,7 +699,7 @@ mod test { let expected = Foo::Variant; let expected_json = json!("Variant"); - let code = "'Variant'"; + let code = c_str!("'Variant'"); test_de(code, &expected, &expected_json); } @@ -604,7 +712,7 @@ mod test { let expected = Foo::Tuple(12, "cat".to_string()); let expected_json = json!({"Tuple": [12, "cat"]}); - let code = "{'Tuple': [12, 'cat']}"; + let code = c_str!("{'Tuple': [12, 'cat']}"); test_de(code, &expected, &expected_json); } @@ -617,7 +725,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!({"NewType": "cat" }); - let code = "{'NewType': 'cat'}"; + let code = c_str!("{'NewType': 'cat'}"); test_de(code, &expected, &expected_json); } @@ -633,7 +741,7 @@ mod test { bar: 25, }; let expected_json = json!({"Struct": {"foo": "cat", "bar": 25 }}); - let code = "{'Struct': {'foo': 'cat', 'bar': 25}}"; + let code = c_str!("{'Struct': {'foo': 'cat', 'bar': 25}}"); test_de(code, &expected, &expected_json); } #[test] @@ -646,7 +754,7 @@ mod test { let expected = Foo::Tuple(12.0, 'c'); let expected_json = json!([12.0, 'c']); - let code = "[12.0, 'c']"; + let code = c_str!("[12.0, 'c']"); test_de(code, &expected, &expected_json); } @@ -660,7 +768,7 @@ mod test { let expected = Foo::NewType("cat".to_string()); let expected_json = json!("cat"); - let code = "'cat'"; + let code = c_str!("'cat'"); test_de(code, &expected, &expected_json); } @@ -677,7 +785,7 @@ mod test { bar: [2, 5, 3, 1], }; let expected_json = json!({"foo": ["a", "b", "c"], "bar": [2, 5, 3, 1]}); - let code = "{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"; + let code = c_str!("{'foo': ['a', 'b', 'c'], 'bar': [2, 5, 3, 1]}"); test_de(code, &expected, &expected_json); } @@ -710,7 +818,75 @@ mod test { }; let expected_json = json!({"name": "SomeFoo", "bar": { "value": 13, "variant": { "Tuple": [-1.5, 8]}}}); - let code = "{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"; + let code = + c_str!("{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"); test_de(code, &expected, &expected_json); } + + #[test] + fn test_int_limits() { + Python::attach(|py| { + // serde_json::Value supports u64 and i64 as maximum sizes + let _: serde_json::Value = depythonize(&u8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u8::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i8::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u16::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i16::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u32::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i32::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: serde_json::Value = depythonize(&u64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&u64::MIN.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: serde_json::Value = depythonize(&i64::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: u128 = depythonize(&u128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&u128::MIN.into_pyobject(py).unwrap()).unwrap(); + + let _: i128 = depythonize(&i128::MAX.into_pyobject(py).unwrap()).unwrap(); + let _: i128 = depythonize(&i128::MIN.into_pyobject(py).unwrap()).unwrap(); + }); + } + + #[test] + fn test_deserialize_bytes() { + Python::attach(|py| { + let obj = PyBytes::new(py, "hello".as_bytes()); + let actual: Vec = depythonize(&obj).unwrap(); + assert_eq!(actual, b"hello"); + }) + } + + #[test] + fn test_char() { + let expected = 'a'; + let expected_json = json!("a"); + let code = c_str!("'a'"); + test_de(code, &expected, &expected_json); + } + + #[test] + fn test_unknown_type() { + Python::attach(|py| { + let obj = py + .import("decimal") + .unwrap() + .getattr("Decimal") + .unwrap() + .call0() + .unwrap(); + let err = depythonize::(&obj).unwrap_err(); + assert!(matches!( + *err.inner, + ErrorImpl::UnsupportedType(name) if name == "Decimal" + )); + }); + } } diff --git a/src/error.rs b/src/error.rs index a86169e..b608106 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ -use pyo3::exceptions::*; -use pyo3::{PyDowncastError, PyErr}; +use pyo3::PyErr; +use pyo3::{exceptions::*, CastError, CastIntoError}; use serde::{de, ser}; +use std::convert::Infallible; use std::error; use std::fmt::{self, Debug, Display}; use std::result; @@ -72,7 +73,7 @@ pub enum ErrorImpl { Message(String), /// A Python type not supported by the deserializer UnsupportedType(String), - /// A `PyAny` object that failed to downcast to an expected Python type + /// A `PyAny` object that failed to cast to an expected Python type UnexpectedType(String), /// Dict keys should be strings to deserialize to struct fields DictKeyNotString, @@ -136,6 +137,13 @@ impl de::Error for PythonizeError { } } +/// Convert an exception raised in Python to a `PythonizeError` +impl From for PythonizeError { + fn from(other: Infallible) -> Self { + match other {} + } +} + /// Convert an exception raised in Python to a `PythonizeError` impl From for PythonizeError { fn from(other: PyErr) -> Self { @@ -145,9 +153,18 @@ impl From for PythonizeError { } } -/// Handle errors that occur when attempting to use `PyAny::cast_as` -impl<'a> From> for PythonizeError { - fn from(other: PyDowncastError) -> Self { +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'a, 'py> From> for PythonizeError { + fn from(other: CastError<'a, 'py>) -> Self { + Self { + inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), + } + } +} + +/// Handle errors that occur when attempting to use `PyAny::cast` +impl<'py> From> for PythonizeError { + fn from(other: CastIntoError<'py>) -> Self { Self { inner: Box::new(ErrorImpl::UnexpectedType(other.to_string())), } diff --git a/src/lib.rs b/src/lib.rs index 139f684..e625b6f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,47 +1,12 @@ -//! This crate converts Rust types which implement the [Serde] serialization -//! traits into Python objects using the [PyO3] library. -//! -//! Pythonize has two public APIs: `pythonize` and `depythonize`. -//! -//! [Serde]: https://github.com/serde-rs/serde -//! [PyO3]: https://github.com/PyO3/pyo3 -//! -//! # Examples -//! ``` -//! use serde::{Serialize, Deserialize}; -//! use pyo3::Python; -//! use pythonize::{depythonize, pythonize}; -//! -//! #[derive(Debug, Serialize, Deserialize, PartialEq)] -//! struct Sample { -//! foo: String, -//! bar: Option -//! } -//! -//! let gil = Python::acquire_gil(); -//! let py = gil.python(); -//! -//! let sample = Sample { -//! foo: "Foo".to_string(), -//! bar: None -//! }; -//! -//! // Rust -> Python -//! let obj = pythonize(py, &sample).unwrap(); -//! -//! assert_eq!("{'foo': 'Foo', 'bar': None}", &format!("{}", obj.as_ref(py).repr().unwrap())); -//! -//! // Python -> Rust -//! let new_sample: Sample = depythonize(obj.as_ref(py)).unwrap(); -//! -//! assert_eq!(new_sample, sample); -//! ``` +#![doc = include_str!("../README.md")] + mod de; mod error; mod ser; -pub use crate::de::depythonize; +pub use crate::de::{depythonize, Depythonizer}; pub use crate::error::{PythonizeError, Result}; pub use crate::ser::{ - pythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, + pythonize, pythonize_custom, PythonizeDefault, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; diff --git a/src/ser.rs b/src/ser.rs index a1f0d86..c8e6dd1 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,207 +1,346 @@ use std::marker::PhantomData; -use pyo3::types::{PyDict, PyList, PyMapping, PySequence, PyTuple}; -use pyo3::{IntoPy, PyObject, PyResult, Python, ToPyObject}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PySequence, PyString, PyTuple, + PyTupleMethods, +}; +use pyo3::{Bound, BoundObject, IntoPyObject, PyAny, PyResult, Python}; use serde::{ser, Serialize}; use crate::error::{PythonizeError, Result}; /// Trait for types which can represent a Python mapping -pub trait PythonizeDictType { - /// Constructor - fn create_mapping(py: Python) -> PyResult<&PyMapping>; +pub trait PythonizeMappingType { + /// Builder type for Python mappings + type Builder<'py>: 'py; + + /// Create a builder for a Python mapping + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult>; + + /// Adds the key-value item to the mapping being built + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()>; + + /// Build the Python mapping + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; +} + +/// Trait for types which can represent a Python mapping and have a name +pub trait PythonizeNamedMappingType { + /// Builder type for Python mappings with a name + type Builder<'py>: 'py; + + /// Create a builder for a Python mapping with a name + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult>; + + /// Adds the field to the named mapping being built + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()>; + + /// Build the Python mapping + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult>; } /// Trait for types which can represent a Python sequence pub trait PythonizeListType: Sized { /// Constructor - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator; } /// Custom types for serialization pub trait PythonizeTypes { /// Python map type (should be representable as python mapping) - type Map: PythonizeDictType; + type Map: PythonizeMappingType; + /// Python (struct-like) named map type (should be representable as python mapping) + type NamedMap: PythonizeNamedMappingType; /// Python sequence type (should be representable as python sequence) type List: PythonizeListType; } -impl PythonizeDictType for PyDict { - fn create_mapping(py: Python) -> PyResult<&PyMapping> { - Ok(PyDict::new(py).as_mapping()) +impl PythonizeMappingType for PyDict { + type Builder<'py> = Bound<'py, Self>; + + fn builder<'py>(py: Python<'py>, _len: Option) -> PyResult> { + Ok(Self::new(py)) + } + + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + builder.set_item(key, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(builder.into_mapping()) + } +} + +/// Adapter type to use an unnamed mapping type, i.e. one that implements +/// [`PythonizeMappingType`], as a named mapping type, i.e. one that implements +/// [`PythonizeNamedMappingType`]. The adapter simply drops the provided name. +/// +/// This adapter is commonly applied to use the same unnamed mapping type for +/// both [`PythonizeTypes::Map`] and [`PythonizeTypes::NamedMap`] while only +/// implementing [`PythonizeMappingType`]. +pub struct PythonizeUnnamedMappingAdapter { + _unnamed: T, +} + +impl PythonizeNamedMappingType for PythonizeUnnamedMappingAdapter { + type Builder<'py> = T::Builder<'py>; + + fn builder<'py>( + py: Python<'py>, + len: usize, + _name: &'static str, + ) -> PyResult> { + T::builder(py, Some(len)) + } + + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + T::push_item(builder, name.into_any(), value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + T::finish(builder) } } impl PythonizeListType for PyList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, + elements: impl IntoIterator, + ) -> PyResult> + where + T: IntoPyObject<'py>, + U: ExactSizeIterator, + { + Ok(PyList::new(py, elements)?.into_sequence()) + } +} + +impl PythonizeListType for PyTuple { + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { - Ok(PyList::new(py, elements).as_sequence()) + Ok(PyTuple::new(py, elements)?.into_sequence()) } } -struct PythonizeDefault; +pub struct PythonizeDefault; impl PythonizeTypes for PythonizeDefault { type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = PyList; } /// Attempt to convert the given data into a Python object -pub fn pythonize(py: Python, value: &T) -> Result +pub fn pythonize<'py, T>(py: Python<'py>, value: &T) -> Result> where T: ?Sized + Serialize, { - pythonize_custom::(py, value) + value.serialize(Pythonizer::new(py)) } /// Attempt to convert the given data into a Python object. /// Also uses custom mapping python class for serialization. -pub fn pythonize_custom(py: Python, value: &T) -> Result +pub fn pythonize_custom<'py, P, T>(py: Python<'py>, value: &T) -> Result> where T: ?Sized + Serialize, P: PythonizeTypes, { - value.serialize(Pythonizer::

{ - py, - _types: PhantomData, - }) + value.serialize(Pythonizer::custom::

(py)) } +/// A structure that serializes Rust values into Python objects #[derive(Clone, Copy)] pub struct Pythonizer<'py, P> { py: Python<'py>, _types: PhantomData

, } +impl<'py, P> From> for Pythonizer<'py, P> { + fn from(py: Python<'py>) -> Self { + Self { + py, + _types: PhantomData, + } + } +} + +impl<'py> Pythonizer<'py, PythonizeDefault> { + /// Creates a serializer to convert data into a Python object using the default mapping class + pub fn new(py: Python<'py>) -> Self { + Self::from(py) + } + + /// Creates a serializer to convert data into a Python object using a custom mapping class + pub fn custom

(py: Python<'py>) -> Pythonizer<'py, P> { + Pythonizer::from(py) + } +} + #[doc(hidden)] pub struct PythonCollectionSerializer<'py, P> { - items: Vec, + items: Vec>, py: Python<'py>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonTupleVariantSerializer<'py, P> { + name: &'static str, variant: &'static str, inner: PythonCollectionSerializer<'py, P>, } #[doc(hidden)] pub struct PythonStructVariantSerializer<'py, P: PythonizeTypes> { + name: &'static str, variant: &'static str, - inner: PythonDictSerializer<'py, P>, + inner: PythonStructDictSerializer<'py, P>, } #[doc(hidden)] -pub struct PythonDictSerializer<'py, P: PythonizeTypes> { +pub struct PythonStructDictSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - dict: &'py PyMapping, + builder: ::Builder<'py>, _types: PhantomData

, } #[doc(hidden)] pub struct PythonMapSerializer<'py, P: PythonizeTypes> { py: Python<'py>, - map: &'py PyMapping, - key: Option, + builder: ::Builder<'py>, + key: Option>, _types: PhantomData

, } +impl<'py, P: PythonizeTypes> Pythonizer<'py, P> { + /// The default implementation for serialisation functions. + #[inline] + fn serialise_default(self, v: T) -> Result> + where + T: IntoPyObject<'py>, + >::Error: Into, + { + v.into_pyobject(self.py) + .map(|x| x.into_any().into_bound()) + .map_err(Into::into) + } +} + impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; type SerializeSeq = PythonCollectionSerializer<'py, P>; type SerializeTuple = PythonCollectionSerializer<'py, P>; type SerializeTupleStruct = PythonCollectionSerializer<'py, P>; type SerializeTupleVariant = PythonTupleVariantSerializer<'py, P>; type SerializeMap = PythonMapSerializer<'py, P>; - type SerializeStruct = PythonDictSerializer<'py, P>; + type SerializeStruct = PythonStructDictSerializer<'py, P>; type SerializeStructVariant = PythonStructVariantSerializer<'py, P>; - fn serialize_bool(self, v: bool) -> Result { - Ok(v.into_py(self.py)) + fn serialize_bool(self, v: bool) -> Result> { + self.serialise_default(v) } - fn serialize_i8(self, v: i8) -> Result { - Ok(v.into_py(self.py)) + fn serialize_i8(self, v: i8) -> Result> { + self.serialise_default(v) } - fn serialize_i16(self, v: i16) -> Result { - Ok(v.into_py(self.py)) + fn serialize_i16(self, v: i16) -> Result> { + self.serialise_default(v) } - fn serialize_i32(self, v: i32) -> Result { - Ok(v.into_py(self.py)) + fn serialize_i32(self, v: i32) -> Result> { + self.serialise_default(v) } - fn serialize_i64(self, v: i64) -> Result { - Ok(v.into_py(self.py)) + fn serialize_i64(self, v: i64) -> Result> { + self.serialise_default(v) } - fn serialize_u8(self, v: u8) -> Result { - Ok(v.into_py(self.py)) + fn serialize_u8(self, v: u8) -> Result> { + self.serialise_default(v) } - fn serialize_u16(self, v: u16) -> Result { - Ok(v.into_py(self.py)) + fn serialize_u16(self, v: u16) -> Result> { + self.serialise_default(v) } - fn serialize_u32(self, v: u32) -> Result { - Ok(v.into_py(self.py)) + fn serialize_u32(self, v: u32) -> Result> { + self.serialise_default(v) } - fn serialize_u64(self, v: u64) -> Result { - Ok(v.into_py(self.py)) + fn serialize_u64(self, v: u64) -> Result> { + self.serialise_default(v) } - fn serialize_f32(self, v: f32) -> Result { - Ok(v.into_py(self.py)) + fn serialize_f32(self, v: f32) -> Result> { + self.serialise_default(v) } - fn serialize_f64(self, v: f64) -> Result { - Ok(v.into_py(self.py)) + fn serialize_f64(self, v: f64) -> Result> { + self.serialise_default(v) } - fn serialize_char(self, v: char) -> Result { + fn serialize_char(self, v: char) -> Result> { self.serialize_str(&v.to_string()) } - fn serialize_str(self, v: &str) -> Result { - Ok(v.into_py(self.py)) + fn serialize_str(self, v: &str) -> Result> { + Ok(PyString::new(self.py, v).into_any()) } - fn serialize_bytes(self, v: &[u8]) -> Result { - Ok(v.into_py(self.py)) + fn serialize_bytes(self, v: &[u8]) -> Result> { + self.serialise_default(v) } - fn serialize_none(self) -> Result { - Ok(self.py.None()) + fn serialize_none(self) -> Result> { + Ok(self.py.None().into_bound(self.py)) } - fn serialize_some(self, value: &T) -> Result + fn serialize_some(self, value: &T) -> Result> where T: ?Sized + Serialize, { value.serialize(self) } - fn serialize_unit(self) -> Result { + fn serialize_unit(self) -> Result> { self.serialize_none() } - fn serialize_unit_struct(self, _name: &'static str) -> Result { + fn serialize_unit_struct(self, _name: &'static str) -> Result> { self.serialize_none() } @@ -210,11 +349,15 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { _name: &'static str, _variant_index: u32, variant: &'static str, - ) -> Result { + ) -> Result> { self.serialize_str(variant) } - fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result> where T: ?Sized + Serialize, { @@ -223,17 +366,21 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_newtype_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, value: &T, - ) -> Result + ) -> Result> where T: ?Sized + Serialize, { - let d = PyDict::new(self.py); - d.set_item(variant, value.serialize(self)?)?; - Ok(d.into()) + let mut m = P::NamedMap::builder(self.py, 1, name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.py, variant), + value.serialize(self)?, + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } fn serialize_seq(self, len: Option) -> Result> { @@ -266,18 +413,22 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_tuple_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, len: usize, ) -> Result> { let inner = self.serialize_tuple(len)?; - Ok(PythonTupleVariantSerializer { variant, inner }) + Ok(PythonTupleVariantSerializer { + name, + variant, + inner, + }) } - fn serialize_map(self, _len: Option) -> Result> { + fn serialize_map(self, len: Option) -> Result> { Ok(PythonMapSerializer { - map: P::Map::create_mapping(self.py)?, + builder: P::Map::builder(self.py, len)?, key: None, py: self.py, _types: PhantomData, @@ -286,28 +437,29 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { fn serialize_struct( self, - _name: &'static str, - _len: usize, - ) -> Result> { - Ok(PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + name: &'static str, + len: usize, + ) -> Result> { + Ok(PythonStructDictSerializer { py: self.py, + builder: P::NamedMap::builder(self.py, len, name)?, _types: PhantomData, }) } fn serialize_struct_variant( self, - _name: &'static str, + name: &'static str, _variant_index: u32, variant: &'static str, - _len: usize, + len: usize, ) -> Result> { Ok(PythonStructVariantSerializer { + name, variant, - inner: PythonDictSerializer { - dict: P::Map::create_mapping(self.py)?, + inner: PythonStructDictSerializer { py: self.py, + builder: P::NamedMap::builder(self.py, len, variant)?, _types: PhantomData, }, }) @@ -315,7 +467,7 @@ impl<'py, P: PythonizeTypes> ser::Serializer for Pythonizer<'py, P> { } impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_element(&mut self, value: &T) -> Result<()> @@ -326,14 +478,14 @@ impl<'py, P: PythonizeTypes> ser::SerializeSeq for PythonCollectionSerializer<'p Ok(()) } - fn end(self) -> Result { + fn end(self) -> Result> { let instance = P::List::create_sequence(self.py, self.items)?; - Ok(instance.to_object(self.py)) + Ok(instance.into_pyobject(self.py)?.into_any()) } } impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_element(&mut self, value: &T) -> Result<()> @@ -343,13 +495,13 @@ impl<'py, P: PythonizeTypes> ser::SerializeTuple for PythonCollectionSerializer< ser::SerializeSeq::serialize_element(self, value) } - fn end(self) -> Result { - Ok(PyTuple::new(self.py, self.items).into()) + fn end(self) -> Result> { + Ok(PyTuple::new(self.py, self.items)?.into_any()) } } impl<'py, P: PythonizeTypes> ser::SerializeTupleStruct for PythonCollectionSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_field(&mut self, value: &T) -> Result<()> @@ -359,13 +511,13 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleStruct for PythonCollectionSeria ser::SerializeSeq::serialize_element(self, value) } - fn end(self) -> Result { + fn end(self) -> Result> { ser::SerializeTuple::end(self) } } impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_field(&mut self, value: &T) -> Result<()> @@ -375,15 +527,19 @@ impl<'py, P: PythonizeTypes> ser::SerializeTupleVariant for PythonTupleVariantSe ser::SerializeSeq::serialize_element(&mut self.inner, value) } - fn end(self) -> Result { - let d = PyDict::new(self.inner.py); - d.set_item(self.variant, ser::SerializeTuple::end(self.inner)?)?; - Ok(d.into()) + fn end(self) -> Result> { + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.inner.py, self.variant), + ser::SerializeTuple::end(self.inner)?, + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } } impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_key(&mut self, key: &T) -> Result<()> @@ -398,7 +554,8 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { where T: ?Sized + Serialize, { - self.map.set_item( + P::Map::push_item( + &mut self.builder, self.key .take() .expect("serialize_value should always be called after serialize_key"), @@ -407,47 +564,57 @@ impl<'py, P: PythonizeTypes> ser::SerializeMap for PythonMapSerializer<'py, P> { Ok(()) } - fn end(self) -> Result { - Ok(self.map.into()) + fn end(self) -> Result> { + Ok(P::Map::finish(self.builder)?.into_any()) } } -impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonDictSerializer<'py, P> { - type Ok = PyObject; +impl<'py, P: PythonizeTypes> ser::SerializeStruct for PythonStructDictSerializer<'py, P> { + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> where T: ?Sized + Serialize, { - Ok(self - .dict - .set_item(key, pythonize_custom::(self.py, value)?)?) + P::NamedMap::push_field( + &mut self.builder, + PyString::new(self.py, key), + pythonize_custom::(self.py, value)?, + )?; + Ok(()) } - fn end(self) -> Result { - Ok(self.dict.into()) + fn end(self) -> Result> { + Ok(P::NamedMap::finish(self.builder)?.into_any()) } } impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariantSerializer<'py, P> { - type Ok = PyObject; + type Ok = Bound<'py, PyAny>; type Error = PythonizeError; fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> where T: ?Sized + Serialize, { - self.inner - .dict - .set_item(key, pythonize_custom::(self.inner.py, value)?)?; + P::NamedMap::push_field( + &mut self.inner.builder, + PyString::new(self.inner.py, key), + pythonize_custom::(self.inner.py, value)?, + )?; Ok(()) } - fn end(self) -> Result { - let d = PyDict::new(self.inner.py); - d.set_item(self.variant, self.inner.dict)?; - Ok(d.into()) + fn end(self) -> Result> { + let v = P::NamedMap::finish(self.inner.builder)?; + let mut m = P::NamedMap::builder(self.inner.py, 1, self.name)?; + P::NamedMap::push_field( + &mut m, + PyString::new(self.inner.py, self.variant), + v.into_any(), + )?; + Ok(P::NamedMap::finish(m)?.into_any()) } } @@ -455,26 +622,29 @@ impl<'py, P: PythonizeTypes> ser::SerializeStructVariant for PythonStructVariant mod test { use super::pythonize; use maplit::hashmap; - use pyo3::types::PyDict; - use pyo3::{PyResult, Python}; - use serde::{Deserialize, Serialize}; + use pyo3::ffi::c_str; + use pyo3::prelude::*; + use pyo3::pybacked::PyBackedStr; + use pyo3::types::{PyBytes, PyDict}; + use serde::Serialize; fn test_ser(src: T, expected: &str) where T: Serialize, { - Python::with_gil(|py| -> PyResult<()> { + Python::attach(|py| -> PyResult<()> { let obj = pythonize(py, &src)?; let locals = PyDict::new(py); locals.set_item("obj", obj)?; py.run( - "import json; result = json.dumps(obj, separators=(',', ':'))", + c_str!("import json; result = json.dumps(obj, separators=(',', ':'))"), None, - Some(locals), + Some(&locals), )?; - let result = locals.get_item("result").unwrap().extract::<&str>()?; + let result = locals.get_item("result")?.unwrap(); + let result = result.extract::()?; assert_eq!(result, expected); assert_eq!(serde_json::to_string(&src).unwrap(), expected); @@ -486,7 +656,7 @@ mod test { #[test] fn test_empty_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct Empty; test_ser(Empty, "null"); @@ -494,7 +664,7 @@ mod test { #[test] fn test_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct Struct { foo: String, bar: usize, @@ -509,9 +679,33 @@ mod test { ); } + #[test] + fn test_nested_struct() { + #[derive(Serialize)] + struct Foo { + name: String, + bar: Bar, + } + + #[derive(Serialize)] + struct Bar { + name: String, + } + + test_ser( + Foo { + name: "foo".to_string(), + bar: Bar { + name: "bar".to_string(), + }, + }, + r#"{"name":"foo","bar":{"name":"bar"}}"#, + ) + } + #[test] fn test_tuple_struct() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] struct TupleStruct(String, usize); test_ser(TupleStruct("foo".to_string(), 5), r#"["foo",5]"#); @@ -534,7 +728,7 @@ mod test { #[test] fn test_enum_unit_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Empty, } @@ -544,7 +738,7 @@ mod test { #[test] fn test_enum_tuple_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Tuple(i32, String), } @@ -554,7 +748,7 @@ mod test { #[test] fn test_enum_newtype_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { NewType(String), } @@ -564,7 +758,7 @@ mod test { #[test] fn test_enum_struct_variant() { - #[derive(Serialize, Deserialize)] + #[derive(Serialize)] enum E { Struct { foo: String, bar: usize }, } @@ -577,4 +771,85 @@ mod test { r#"{"Struct":{"foo":"foo","bar":5}}"#, ); } + + #[test] + fn test_integers() { + #[derive(Serialize)] + struct Integers { + a: i8, + b: i16, + c: i32, + d: i64, + e: u8, + f: u16, + g: u32, + h: u64, + } + + test_ser( + Integers { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + f: 6, + g: 7, + h: 8, + }, + r#"{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8}"#, + ) + } + + #[test] + fn test_floats() { + #[derive(Serialize)] + struct Floats { + a: f32, + b: f64, + } + + test_ser(Floats { a: 1.0, b: 2.0 }, r#"{"a":1.0,"b":2.0}"#) + } + + #[test] + fn test_char() { + #[derive(Serialize)] + struct Char { + a: char, + } + + test_ser(Char { a: 'a' }, r#"{"a":"a"}"#) + } + + #[test] + fn test_bool() { + test_ser(true, "true"); + test_ser(false, "false"); + } + + #[test] + fn test_none() { + #[derive(Serialize)] + struct S; + + test_ser((), "null"); + test_ser(S, "null"); + + test_ser(Some(1), "1"); + test_ser(None::, "null"); + } + + #[test] + fn test_bytes() { + // serde treats &[u8] as a sequence of integers due to lack of specialization + test_ser(b"foo", "[102,111,111]"); + + Python::attach(|py| { + assert!(pythonize(py, serde_bytes::Bytes::new(b"foo")) + .expect("bytes will always serialize successfully") + .eq(&PyBytes::new(py, b"foo")) + .expect("bytes will always compare successfully")); + }); + } } diff --git a/tests/test_custom_types.rs b/tests/test_custom_types.rs index ff437cc..27888d0 100644 --- a/tests/test_custom_types.rs +++ b/tests/test_custom_types.rs @@ -3,16 +3,19 @@ use std::collections::HashMap; use pyo3::{ exceptions::{PyIndexError, PyKeyError}, prelude::*, - types::{PyDict, PyList, PyMapping, PySequence}, + types::{PyDict, PyMapping, PySequence, PyTuple}, + IntoPyObjectExt, }; use pythonize::{ - depythonize, pythonize_custom, PythonizeDictType, PythonizeListType, PythonizeTypes, + depythonize, pythonize_custom, PythonizeListType, PythonizeMappingType, + PythonizeNamedMappingType, PythonizeTypes, PythonizeUnnamedMappingAdapter, Pythonizer, }; +use serde::Serialize; use serde_json::{json, Value}; -#[pyclass] +#[pyclass(sequence)] struct CustomList { - items: Vec, + items: Vec>, } #[pymethods] @@ -21,7 +24,7 @@ impl CustomList { self.items.len() } - fn __getitem__(&self, idx: isize) -> PyResult { + fn __getitem__(&self, idx: isize) -> PyResult> { self.items .get(idx as usize) .cloned() @@ -30,51 +33,50 @@ impl CustomList { } impl PythonizeListType for CustomList { - fn create_sequence( - py: Python, + fn create_sequence<'py, T, U>( + py: Python<'py>, elements: impl IntoIterator, - ) -> PyResult<&PySequence> + ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'py>, U: ExactSizeIterator, { - let sequence = Py::new( + let sequence = Bound::new( py, CustomList { items: elements .into_iter() - .map(|item| item.to_object(py)) - .collect(), + .map(|item| item.into_py_any(py)) + .collect::>()?, }, - )? - .into_ref(py); + )?; - Ok(unsafe { PySequence::try_from_unchecked(sequence.as_ref()) }) + Ok(unsafe { sequence.cast_into_unchecked() }) } } struct PythonizeCustomList; -impl PythonizeTypes for PythonizeCustomList { +impl<'py> PythonizeTypes for PythonizeCustomList { type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; type List = CustomList; } #[test] fn test_custom_list() { - Python::with_gil(|py| { - let serialized = pythonize_custom::(py, &json!([1, 2, 3])) - .unwrap() - .into_ref(py); - assert!(serialized.is_instance_of::().unwrap()); + Python::attach(|py| { + PySequence::register::(py).unwrap(); + let serialized = pythonize_custom::(py, &json!([1, 2, 3])).unwrap(); + assert!(serialized.is_instance_of::()); - let deserialized: Value = depythonize(serialized).unwrap(); + let deserialized: Value = depythonize(&serialized).unwrap(); assert_eq!(deserialized, json!([1, 2, 3])); }) } #[pyclass(mapping)] struct CustomDict { - items: HashMap, + items: HashMap>, } #[pymethods] @@ -83,14 +85,14 @@ impl CustomDict { self.items.len() } - fn __getitem__(&self, key: String) -> PyResult { + fn __getitem__(&self, key: String) -> PyResult> { self.items .get(&key) .cloned() .ok_or_else(|| PyKeyError::new_err(key)) } - fn __setitem__(&mut self, key: String, value: PyObject) { + fn __setitem__(&mut self, key: String, value: Py) { self.items.insert(key, value); } @@ -98,40 +100,186 @@ impl CustomDict { self.items.keys().collect() } - fn values(&self) -> Vec { + fn values(&self) -> Vec> { self.items.values().cloned().collect() } } -impl PythonizeDictType for CustomDict { - fn create_mapping(py: Python) -> PyResult<&PyMapping> { - let mapping = Py::new( +impl PythonizeMappingType for CustomDict { + type Builder<'py> = Bound<'py, CustomDict>; + + fn builder<'py>(py: Python<'py>, len: Option) -> PyResult> { + Bound::new( py, CustomDict { - items: HashMap::new(), + items: HashMap::with_capacity(len.unwrap_or(0)), }, - )? - .into_ref(py); - Ok(unsafe { PyMapping::try_from_unchecked(mapping.as_ref()) }) + ) + } + + fn push_item<'py>( + builder: &mut Self::Builder<'py>, + key: Bound<'py, PyAny>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + unsafe { builder.cast_unchecked::() }.set_item(key, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(unsafe { builder.cast_into_unchecked() }) } } struct PythonizeCustomDict; -impl PythonizeTypes for PythonizeCustomDict { +impl<'py> PythonizeTypes for PythonizeCustomDict { type Map = CustomDict; - type List = PyList; + type NamedMap = PythonizeUnnamedMappingAdapter; + type List = PyTuple; } #[test] fn test_custom_dict() { - Python::with_gil(|py| { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); let serialized = pythonize_custom::(py, &json!({ "hello": 1, "world": 2 })) - .unwrap() - .into_ref(py); - assert!(serialized.is_instance_of::().unwrap()); + .unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} + +#[test] +fn test_tuple() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &json!([1, 2, 3, 4])).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!([1, 2, 3, 4])); + }) +} + +#[test] +fn test_pythonizer_can_be_created() { + // https://github.com/davidhewitt/pythonize/pull/56 + Python::attach(|py| { + let sample = json!({ "hello": 1, "world": 2 }); + assert!(sample + .serialize(Pythonizer::new(py)) + .unwrap() + .is_instance_of::()); + + assert!(sample + .serialize(Pythonizer::custom::(py)) + .unwrap() + .is_instance_of::()); + }) +} + +#[pyclass(mapping)] +struct NamedCustomDict { + name: String, + items: HashMap>, +} + +#[pymethods] +impl NamedCustomDict { + fn __len__(&self) -> usize { + self.items.len() + } + + fn __getitem__(&self, key: String) -> PyResult> { + self.items + .get(&key) + .cloned() + .ok_or_else(|| PyKeyError::new_err(key)) + } + + fn __setitem__(&mut self, key: String, value: Py) { + self.items.insert(key, value); + } + + fn keys(&self) -> Vec<&String> { + self.items.keys().collect() + } + + fn values(&self) -> Vec> { + self.items.values().cloned().collect() + } +} + +impl PythonizeNamedMappingType for NamedCustomDict { + type Builder<'py> = Bound<'py, NamedCustomDict>; + + fn builder<'py>( + py: Python<'py>, + len: usize, + name: &'static str, + ) -> PyResult> { + Bound::new( + py, + NamedCustomDict { + name: String::from(name), + items: HashMap::with_capacity(len), + }, + ) + } + + fn push_field<'py>( + builder: &mut Self::Builder<'py>, + name: Bound<'py, pyo3::types::PyString>, + value: Bound<'py, PyAny>, + ) -> PyResult<()> { + unsafe { builder.cast_unchecked::() }.set_item(name, value) + } + + fn finish<'py>(builder: Self::Builder<'py>) -> PyResult> { + Ok(unsafe { builder.cast_into_unchecked() }) + } +} + +struct PythonizeNamedCustomDict; +impl<'py> PythonizeTypes for PythonizeNamedCustomDict { + type Map = CustomDict; + type NamedMap = NamedCustomDict; + type List = PyTuple; +} + +#[derive(Serialize)] +struct Struct { + hello: u8, + world: i8, +} + +#[test] +fn test_custom_unnamed_dict() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }).unwrap(); + assert!(serialized.is_instance_of::()); + + let deserialized: Value = depythonize(&serialized).unwrap(); + assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); + }) +} + +#[test] +fn test_custom_named_dict() { + Python::attach(|py| { + PyMapping::register::(py).unwrap(); + let serialized = + pythonize_custom::(py, &Struct { hello: 1, world: 2 }) + .unwrap(); + let named: Bound = serialized.extract().unwrap(); + assert_eq!(named.borrow().name, "Struct"); - let deserialized: Value = depythonize(serialized).unwrap(); + let deserialized: Value = depythonize(&serialized).unwrap(); assert_eq!(deserialized, json!({ "hello": 1, "world": 2 })); }) } diff --git a/tests/test_with_serde_path_to_err.rs b/tests/test_with_serde_path_to_err.rs new file mode 100644 index 0000000..82fd8bb --- /dev/null +++ b/tests/test_with_serde_path_to_err.rs @@ -0,0 +1,211 @@ +use std::collections::BTreeMap; + +use pyo3::{ + prelude::*, + types::{PyDict, PyList}, +}; +use pythonize::{PythonizeTypes, PythonizeUnnamedMappingAdapter}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +struct Root { + root_key: String, + root_map: BTreeMap>, +} + +impl<'py, T> PythonizeTypes for Root { + type Map = PyDict; + type NamedMap = PythonizeUnnamedMappingAdapter; + type List = PyList; +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +struct Nested { + nested_key: T, +} + +#[derive(Deserialize, Debug, PartialEq, Eq)] +struct CannotSerialize {} + +impl Serialize for CannotSerialize { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + Err(serde::ser::Error::custom( + "something went intentionally wrong", + )) + } +} + +#[test] +fn test_de_valid() { + Python::attach(|py| { + let pyroot = PyDict::new(py); + pyroot.set_item("root_key", "root_value").unwrap(); + + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); + nested_0.set_item("nested_key", "nested_value_0").unwrap(); + nested.set_item("nested_0", nested_0).unwrap(); + let nested_1 = PyDict::new(py); + nested_1.set_item("nested_key", "nested_value_1").unwrap(); + nested.set_item("nested_1", nested_1).unwrap(); + + pyroot.set_item("root_map", nested).unwrap(); + + let de = &mut pythonize::Depythonizer::from_object(&pyroot); + let root: Root = serde_path_to_error::deserialize(de).unwrap(); + + assert_eq!( + root, + Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: String::from("nested_value_0") + } + ), + ( + String::from("nested_1"), + Nested { + nested_key: String::from("nested_value_1") + } + ) + ]) + } + ); + }) +} + +#[test] +fn test_de_invalid() { + Python::attach(|py| { + let pyroot = PyDict::new(py); + pyroot.set_item("root_key", "root_value").unwrap(); + + let nested = PyDict::new(py); + let nested_0 = PyDict::new(py); + nested_0.set_item("nested_key", "nested_value_0").unwrap(); + nested.set_item("nested_0", nested_0).unwrap(); + let nested_1 = PyDict::new(py); + nested_1.set_item("nested_key", 1).unwrap(); + nested.set_item("nested_1", nested_1).unwrap(); + + pyroot.set_item("root_map", nested).unwrap(); + + let de = &mut pythonize::Depythonizer::from_object(&pyroot); + let err = serde_path_to_error::deserialize::<_, Root>(de).unwrap_err(); + + assert_eq!(err.path().to_string(), "root_map.nested_1.nested_key"); + assert_eq!( + err.to_string(), + "root_map.nested_1.nested_key: unexpected type: 'int' object cannot be cast as 'str'" + ); + }) +} + +#[test] +fn test_ser_valid() { + Python::attach(|py| { + let root = Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: String::from("nested_value_0"), + }, + ), + ( + String::from("nested_1"), + Nested { + nested_key: String::from("nested_value_1"), + }, + ), + ]), + }; + + let ser = pythonize::Pythonizer::>::from(py); + let pyroot: Bound<'_, PyAny> = serde_path_to_error::serialize(&root, ser).unwrap(); + + let pyroot = pyroot.cast::().unwrap(); + assert_eq!(pyroot.len(), 2); + + let root_value: String = pyroot + .get_item("root_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(root_value, "root_value"); + + let root_map = pyroot + .get_item("root_map") + .unwrap() + .unwrap() + .cast_into::() + .unwrap(); + assert_eq!(root_map.len(), 2); + + let nested_0 = root_map + .get_item("nested_0") + .unwrap() + .unwrap() + .cast_into::() + .unwrap(); + assert_eq!(nested_0.len(), 1); + let nested_key_0: String = nested_0 + .get_item("nested_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_key_0, "nested_value_0"); + + let nested_1 = root_map + .get_item("nested_1") + .unwrap() + .unwrap() + .cast_into::() + .unwrap(); + assert_eq!(nested_1.len(), 1); + let nested_key_1: String = nested_1 + .get_item("nested_key") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(nested_key_1, "nested_value_1"); + }); +} + +#[test] +fn test_ser_invalid() { + Python::attach(|py| { + let root = Root { + root_key: String::from("root_value"), + root_map: BTreeMap::from([ + ( + String::from("nested_0"), + Nested { + nested_key: CannotSerialize {}, + }, + ), + ( + String::from("nested_1"), + Nested { + nested_key: CannotSerialize {}, + }, + ), + ]), + }; + + let ser = pythonize::Pythonizer::>::from(py); + let err = serde_path_to_error::serialize(&root, ser).unwrap_err(); + + assert_eq!(err.path().to_string(), "root_map.nested_0.nested_key"); + }); +}