fix: sanitize big-ints in table manager (#4468)

Fixes #4465

Author: mscolnick

marimo/_output/data/data.py

@@ -3,7 +3,7 @@
 
 import base64
 import io
-from typing import Union
+from typing import Any, Union
 
 from marimo._plugins.core.media import is_data_empty
 from marimo._runtime.virtual_file import (
@@ -171,3 +171,37 @@ def any_data(data: Union[str, bytes, io.BytesIO], ext: str) -> VirtualFile:
         return item.virtual_file
 
     raise ValueError(f"Unsupported data type: {type(data)}")
+
+
+def sanitize_json_bigint(
+    data: Union[str, dict[str, Any], list[dict[str, Any]]],
+) -> str:
+    """Sanitize JSON bigint to a string.
+
+    This is necessary because the frontend will round ints larger than
+    Number.MAX_SAFE_INTEGER to Number.MAX_SAFE_INTEGER.
+    """
+    from json import dumps, loads
+
+    # JavaScript's safe integer limits
+    MAX_SAFE_INTEGER = 9007199254740991
+    MIN_SAFE_INTEGER = -9007199254740991
+
+    def convert_bigint(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: convert_bigint(v) for k, v in obj.items()}  # type: ignore
+        elif isinstance(obj, list):
+            return [convert_bigint(item) for item in obj]  # type: ignore
+        elif isinstance(obj, int) and (
+            obj > MAX_SAFE_INTEGER or obj < MIN_SAFE_INTEGER
+        ):
+            return str(obj)
+        else:
+            return obj
+
+    if isinstance(data, str):
+        as_json = loads(data)
+    else:
+        as_json = data
+
+    return dumps(convert_bigint(as_json), indent=None, separators=(",", ":"))

marimo/_plugins/ui/_impl/tables/default_table.py

@@ -88,20 +88,24 @@ def to_data(
     ) -> JSONType:
         return self._normalize_data(self.apply_formatting(format_mapping).data)
 
-    def to_csv(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
+    def to_csv_str(
+        self, format_mapping: Optional[FormatMapping] = None
+    ) -> str:
         if isinstance(self.data, dict) and not self.is_column_oriented:
-            return DefaultTableManager(self._normalize_data(self.data)).to_csv(
-                format_mapping
-            )
+            return DefaultTableManager(
+                self._normalize_data(self.data)
+            ).to_csv_str(format_mapping)
 
-        return self._as_table_manager().to_csv(format_mapping)
+        return self._as_table_manager().to_csv_str(format_mapping)
 
-    def to_json(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
+    def to_json_str(
+        self, format_mapping: Optional[FormatMapping] = None
+    ) -> str:
         if isinstance(self.data, dict) and not self.is_column_oriented:
             return DefaultTableManager(
                 self._normalize_data(self.data)
-            ).to_json(format_mapping)
-        return self._as_table_manager().to_json(format_mapping)
+            ).to_json_str(format_mapping)
+        return self._as_table_manager().to_json_str(format_mapping)
 
     def select_rows(self, indices: list[int]) -> DefaultTableManager:
         if isinstance(self.data, dict):

marimo/_plugins/ui/_impl/tables/ibis_table.py

@@ -40,15 +40,15 @@ def create() -> type[TableManager[Any]]:
         class IbisTableManager(TableManager[ibis.Table]):
             type = "ibis"
 
-            def to_csv(
+            def to_csv_str(
                 self, format_mapping: Optional[FormatMapping] = None
-            ) -> bytes:
-                return self._as_table_manager().to_csv(format_mapping)
+            ) -> str:
+                return self._as_table_manager().to_csv_str(format_mapping)
 
-            def to_json(
+            def to_json_str(
                 self, format_mapping: Optional[FormatMapping] = None
-            ) -> bytes:
-                return self._as_table_manager().to_json(format_mapping)
+            ) -> str:
+                return self._as_table_manager().to_json_str(format_mapping)
 
             def supports_download(self) -> bool:
                 return False

marimo/_plugins/ui/_impl/tables/narwhals_table.py

@@ -1,7 +1,6 @@
 # Copyright 2024 Marimo. All rights reserved.
 from __future__ import annotations
 
-import json
 from functools import cached_property
 from typing import Any, Optional, Union, cast
 
@@ -11,6 +10,7 @@
 from marimo import _loggers
 from marimo._data.models import ColumnSummary, ExternalDataType
 from marimo._dependencies.dependencies import DependencyManager
+from marimo._output.data.data import sanitize_json_bigint
 from marimo._plugins.core.media import io_to_data_url
 from marimo._plugins.ui._impl.tables.format import (
     FormatMapping,
@@ -60,28 +60,28 @@ def with_new_data(
         # of the subclass with the native data.
         return self.__class__(data.to_native())
 
-    def to_csv(
+    def to_csv_str(
         self,
         format_mapping: Optional[FormatMapping] = None,
-    ) -> bytes:
+    ) -> str:
         _data = self.apply_formatting(format_mapping).as_frame()
-        return dataframe_to_csv(_data).encode("utf-8")
+        return dataframe_to_csv(_data)
 
-    def to_json(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
+    def to_json_str(
+        self, format_mapping: Optional[FormatMapping] = None
+    ) -> str:
         try:
-            csv_str = self.to_csv(format_mapping=format_mapping).decode(
-                "utf-8"
-            )
+            csv_str = self.to_csv_str(format_mapping=format_mapping)
         except Exception as e:
             LOGGER.debug(
                 f"Failed to use format mapping: {str(e)}, falling back to default"
             )
-            csv_str = self.to_csv().decode("utf-8")
+            csv_str = self.to_csv_str()
 
         import csv
 
         csv_reader = csv.DictReader(csv_str.splitlines())
-        return json.dumps([row for row in csv_reader]).encode("utf-8")
+        return sanitize_json_bigint([row for row in csv_reader])
 
     def apply_formatting(
         self, format_mapping: Optional[FormatMapping]

marimo/_plugins/ui/_impl/tables/pandas_table.py

@@ -9,6 +9,7 @@
 
 from marimo import _loggers
 from marimo._data.models import ExternalDataType
+from marimo._output.data.data import sanitize_json_bigint
 from marimo._plugins.ui._impl.tables.format import (
     FormatMapping,
     format_value,
@@ -44,30 +45,28 @@ def __init__(self, data: pd.DataFrame) -> None:
             def schema(self) -> pd.Series[Any]:
                 return self._original_data.dtypes  # type: ignore
 
-            # We override narwhals's to_csv to handle pandas
+            # We override narwhals's to_csv_str to handle pandas
             # headers
-            def to_csv(
+            def to_csv_str(
                 self, format_mapping: Optional[FormatMapping] = None
-            ) -> bytes:
+            ) -> str:
                 has_headers = len(self.get_row_headers()) > 0
                 # Pandas omits H:M:S for datetimes when H:M:S is identically
                 # 0; this doesn't play well with our frontend table component,
                 # so we use an explicit date format.
-                return (
-                    self.apply_formatting(format_mapping)
-                    ._original_data.to_csv(
-                        # By adding %H:%M:%S and %z, we ensure that the
-                        # datetime is displayed in the frontend with the
-                        # correct timezone.
-                        index=has_headers,
-                        date_format="%Y-%m-%d %H:%M:%S%z",
-                    )
-                    .encode("utf-8")
+                return self.apply_formatting(
+                    format_mapping
+                )._original_data.to_csv(
+                    # By adding %H:%M:%S and %z, we ensure that the
+                    # datetime is displayed in the frontend with the
+                    # correct timezone.
+                    index=has_headers,
+                    date_format="%Y-%m-%d %H:%M:%S%z",
                 )
 
-            def to_json(
+            def to_json_str(
                 self, format_mapping: Optional[FormatMapping] = None
-            ) -> bytes:
+            ) -> str:
                 from pandas.api.types import (
                     is_complex_dtype,
                     is_object_dtype,
@@ -98,7 +97,9 @@ def to_json(
                         "Error handling complex or timedelta64 dtype",
                         exc_info=e,
                     )
-                    return result.to_json(orient="records").encode("utf-8")
+                    return sanitize_json_bigint(
+                        result.to_json(orient="records")
+                    )
 
                 # Flatten row multi-index
                 if isinstance(result.index, pd.MultiIndex) or (
@@ -120,10 +121,9 @@ def to_json(
                                 "Indexes with more than one level are not supported properly, call reset_index() to flatten"
                             )
 
-                return result.to_json(
-                    orient="records",
-                    date_format="iso",
-                ).encode("utf-8")
+                return sanitize_json_bigint(
+                    result.to_json(orient="records", date_format="iso")
+                )
 
             def to_arrow_ipc(self) -> bytes:
                 out = io.BytesIO()

marimo/_plugins/ui/_impl/tables/polars_table.py

@@ -11,6 +11,7 @@
 from marimo._data.models import (
     ExternalDataType,
 )
+from marimo._output.data.data import sanitize_json_bigint
 from marimo._plugins.ui._impl.tables.format import (
     FormatMapping,
     format_value,
@@ -64,13 +65,13 @@ def to_arrow_ipc(self) -> bytes:
 
             # We override narwhals's to_csv to handle polars
             # nested data types.
-            def to_csv(
+            def to_csv_str(
                 self,
                 format_mapping: Optional[FormatMapping] = None,
-            ) -> bytes:
+            ) -> str:
                 _data = self.apply_formatting(format_mapping).collect()
                 try:
-                    return _data.write_csv().encode("utf-8")
+                    return _data.write_csv()
                 except pl.exceptions.ComputeError:
                     # Likely CSV format does not support nested data or objects
                     # Try to convert columns to json or strings
@@ -99,11 +100,11 @@ def to_csv(
                             result = self._convert_time_to_string(
                                 result, column
                             )
-                    return result.write_csv().encode("utf-8")
+                    return result.write_csv()
 
-            def to_json(
+            def to_json_str(
                 self, format_mapping: Optional[FormatMapping] = None
-            ) -> bytes:
+            ) -> str:
                 result = self.apply_formatting(format_mapping).collect()
                 try:
                     for column in result.get_columns():
@@ -112,7 +113,7 @@ def to_json(
                             result = self._convert_time_to_string(
                                 result, column
                             )
-                    return result.write_json().encode("utf-8")
+                    return sanitize_json_bigint(result.write_json())
                 except (
                     BaseException
                 ):  # Sometimes, polars throws a generic exception
@@ -140,7 +141,7 @@ def to_json(
                                 result, column
                             )
 
-                    return result.write_json().encode("utf-8")
+                    return sanitize_json_bigint(result.write_json())
 
             def _convert_time_to_string(
                 self, result: pl.DataFrame, column: pl.Series

marimo/_plugins/ui/_impl/tables/table_manager.py

@@ -94,19 +94,30 @@ def sort_values(
         pass
 
     @abc.abstractmethod
+    def to_csv_str(
+        self,
+        format_mapping: Optional[FormatMapping] = None,
+    ) -> str:
+        pass
+
     def to_csv(
         self,
         format_mapping: Optional[FormatMapping] = None,
     ) -> bytes:
-        pass
+        return self.to_csv_str(format_mapping).encode("utf-8")
 
     def to_arrow_ipc(self) -> bytes:
         raise NotImplementedError("Arrow format not supported")
 
     @abc.abstractmethod
-    def to_json(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
+    def to_json_str(
+        self, format_mapping: Optional[FormatMapping] = None
+    ) -> str:
         pass
 
+    def to_json(self, format_mapping: Optional[FormatMapping] = None) -> bytes:
+        return self.to_json_str(format_mapping).encode("utf-8")
+
     @abc.abstractmethod
     def select_rows(self, indices: list[int]) -> TableManager[Any]:
         pass

marimo/_smoke_tests/tables/big_ints.py

@@ -0,0 +1,61 @@
+
+
+import marimo
+
+__generated_with = "0.12.7"
+app = marimo.App(width="medium")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""## Big Ints""")
+    return
+
+
+@app.cell(hide_code=True)
+def _():
+    data = {
+        "int": [1, 2, 3],
+        "bigint_1": [
+            1000000000000000000,
+            1000000000000000001,
+            1000000000000000002,
+        ],
+        "bigint_2": [
+            2000000000000000000,
+            2000000000000000001,
+            2000000000000000002,
+        ],
+        "bigint_3": [
+            3000000000000000000,
+            3000000000000000001,
+            3000000000000000002,
+        ],
+    }
+    return (data,)
+
+
+@app.cell
+def _(data):
+    import pandas as pd
+
+    pd.DataFrame(data)
+    return
+
+
+@app.cell
+def _(data):
+    import polars as pl
+
+    pl.DataFrame(data)
+    return
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+if __name__ == "__main__":
+    app.run()