Store null in review_result column during review phase (LAION-AI#1599)

Author: andreaskoepf

backend/alembic/versions/2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py

@@ -0,0 +1,36 @@
+"""message review_result nullable
+
+Revision ID: 8cd0c34d0c3c
+Revises: 165b55de5a94
+Create Date: 2023-02-15 17:54:58.029278
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8cd0c34d0c3c"
+down_revision = "165b55de5a94"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "message",
+        "review_result",
+        existing_type=sa.BOOLEAN(),
+        nullable=True,
+        server_default=None,
+        existing_server_default=sa.text("false"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "message", "review_result", existing_type=sa.BOOLEAN(), nullable=False, server_default=sa.text("false")
+    )
+    # ### end Alembic commands ###

backend/oasst_backend/models/message.py

@@ -52,7 +52,7 @@ def __new__(cls, *args: Any, **kwargs: Any):
     deleted: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=False, server_default=false()))
 
     review_count: int = Field(sa_column=sa.Column(sa.Integer, default=0, server_default=sa.text("0"), nullable=False))
-    review_result: bool = Field(sa_column=sa.Column(sa.Boolean, default=False, server_default=false(), nullable=False))
+    review_result: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=True))
     ranking_count: int = Field(sa_column=sa.Column(sa.Integer, default=0, server_default=sa.text("0"), nullable=False))
 
     rank: Optional[int] = Field(nullable=True)

backend/oasst_backend/prompt_repository.py

@@ -132,7 +132,7 @@ def insert_message(
         payload_type: str = None,
         depth: int = 0,
         review_count: int = 0,
-        review_result: bool = False,
+        review_result: bool = None,
     ) -> Message:
         if payload_type is None:
             if payload is None:
@@ -198,7 +198,7 @@ def store_text_reply(
         frontend_message_id: str,
         user_frontend_message_id: str,
         review_count: int = 0,
-        review_result: bool = False,
+        review_result: bool = None,
         check_tree_state: bool = True,
         check_duplicate: bool = True,
     ) -> Message:

backend/oasst_backend/tree_manager.py

@@ -742,6 +742,8 @@ async def handle_interaction(self, interaction: protocol_schema.AnyInteraction)
                                     f"Initial prompt message was accepted: {msg.id=}, {acceptance_score=}, {len(reviews)=}"
                                 )
                             else:
+                                msg.review_result = False
+                                self.db.add(msg)
                                 self.enter_low_grade_state(msg.message_tree_id)
                         self.check_condition_for_prompt_lottery(msg.message_tree_id)
                     elif msg.review_count >= self.cfg.num_reviews_reply:
@@ -751,6 +753,9 @@ async def handle_interaction(self, interaction: protocol_schema.AnyInteraction)
                             logger.info(
                                 f"Reply message message accepted: {msg.id=}, {acceptance_score=}, {len(reviews)=}"
                             )
+                        else:
+                            msg.review_result = False
+                            self.db.add(msg)
 
                     self.check_condition_for_ranking_state(msg.message_tree_id)
 
@@ -965,7 +970,7 @@ def _query_need_review(
             .filter(
                 MessageTreeState.active,
                 MessageTreeState.state == state,
-                not_(Message.review_result),
+                or_(Message.review_result.is_(None), not_(Message.review_result)),
                 not_(Message.deleted),
                 Message.review_count < required_reviews,
                 Message.lang == lang,
@@ -1183,7 +1188,10 @@ def query_tree_size(self, message_tree_id: UUID) -> ActiveTreeSizeRow:
                 MessageTreeState.goal_tree_size.label("goal_tree_size"),
                 func.count(Message.id).filter(Message.review_result).label("tree_size"),
                 func.count(Message.id)
-                .filter(not_(Message.review_result), Message.review_count < required_reviews)
+                .filter(
+                    or_(Message.review_result.is_(None), not_(Message.review_result)),
+                    Message.review_count < required_reviews,
+                )
                 .label("awaiting_review"),
             )
             .select_from(MessageTreeState)