feat: save raw data of lc problems

Author: yanglbme

.gitignore

@@ -1,8 +1,10 @@
-.idea/
-.DS_Store
-.vscode
-/node_modules
-/solution/result.json
-/lcof/lcof.json
-/lcof/lcof_list.json
-/lcci/lcci.json
+.idea/
+.DS_Store
+.vscode
+/node_modules
+/solution/result.json
+/solution/raw.json
+/lcof/lcof.json
+/lcof/lcof_list.json
+/lcci/lcci.json
+/solution/__pycache__

solution/main.py

@@ -1,6 +1,7 @@
 import json
 import os
-from urllib.parse import quote
+import re
+from urllib.parse import quote, unquote
 
 from spider import Spider
 
@@ -130,6 +131,66 @@ def generate_summary(result):
         f.write(summary_en)
 
 
+def refresh(result):
+    """update problems"""
+    pattern = re.compile("src=\"(.*?)\"")
+
+    for question in result:
+        front_question_id = question['frontend_question_id']
+        print(front_question_id)
+
+        path_cn = unquote(str(question['relative_path_cn']).replace("/solution", "."))
+        path_en = unquote(str(question['relative_path_en']).replace("/solution", "."))
+
+        with open(path_cn, 'r', encoding='utf-8') as f1:
+            cn_content = f1.read()
+
+        with open(path_en, 'r', encoding='utf-8') as f2:
+            en_content = f2.read()
+
+        # update question content
+        old_content = re.search("<!-- 这里写题目描述 -->(.*?)## 解法", cn_content, re.S).group(1)
+        cn_content = cn_content.replace(
+            old_content, "\n\n" + question['content_cn'] + "\n\n"
+        ).replace("\n\n    <ul>", "\n    <ul>")
+
+        # replace image url to cdn link
+        for url in pattern.findall(cn_content) or []:
+            image_name = (
+                os.path.basename(url).replace('.PNG', '.png').replace('.JPG', '.jpg')
+            )
+            new_url = (
+                'https://cdn.jsdelivr.net/gh/doocs/leetcode@main'
+                + str(question['relative_path_cn']).replace("README.md", "images/")
+                + image_name
+            )
+            cn_content = cn_content.replace(url, new_url)
+
+        with open(path_cn, 'w', encoding='utf-8') as f1:
+            f1.write(cn_content)
+
+        old_content = re.search(
+            "## Description(.*?)## Solutions", en_content, re.S
+        ).group(1)
+        en_content = en_content.replace(
+            old_content, "\n\n" + question['content_en'] + "\n\n"
+        ).replace("\n\n    <ul>", "\n    <ul>")
+
+        for url in pattern.findall(en_content) or []:
+            image_name = (
+                os.path.basename(url).replace('.PNG', '.png').replace('.JPG', '.jpg')
+            )
+            new_url = (
+                'https://cdn.jsdelivr.net/gh/doocs/leetcode@main'
+                + str(question['relative_path_cn']).replace("README.md", "images/")
+                + image_name
+            )
+            en_content = en_content.replace(url, new_url)
+
+        with open(path_en, 'w', encoding='utf-8') as f2:
+            f2.write(en_content)
+
+
 def save(result):
     with open('./result.json', 'w', encoding='utf-8') as f:
         f.write(json.dumps(result))
@@ -140,6 +201,7 @@ def save(result):
     cookie_en = ''
     spider = Spider(cookie_cn, cookie_en)
     res = spider.run()
+    save(res)
 
     # with open('./result.json', 'r', encoding='utf-8') as f:
     #     res = f.read()
@@ -148,4 +210,4 @@ def save(result):
     generate_readme(res)
     generate_question_readme(res)
     generate_summary(res)
-    save(res)
+    # refresh(res)

solution/refresh.py

@@ -90,6 +90,7 @@ def __init__(self, cookie_cn: str, cookie_en: str):
         self.cookie_cn = cookie_cn
         self.cookie_en = cookie_en
         self.session = requests.session()
+        self.raw_data = {}
 
     def get_all_questions(self) -> List:
         """获取所有题目"""
@@ -192,6 +193,7 @@ def handle(self, question: dict) -> dict:
         url_cn = f'https://leetcode-cn.com/problems/{question_title_slug}'
         url_en = f'https://leetcode.com/problems/{question_title_slug}'
         frontend_question_id = str(question['stat']['frontend_question_id']).zfill(4)
+        self.raw_data[frontend_question_id] = question_detail
         no = int(frontend_question_id) // 100
         question_title_en = question['stat']['question__title']
         question_title_en = re.sub(r'[\\/:*?"<>|]', '', question_title_en).strip()
@@ -251,6 +253,10 @@ def handle(self, question: dict) -> dict:
         item['md_table_row_en'] = [col1_en, col2_en, col3_en, col4_en, col5_en]
         return item
 
+    def save(self):
+        with open('./raw.json', 'w', encoding='utf-8') as f:
+            f.write(json.dumps(self.raw_data))
+
     def run(self):
         questions = self.get_all_questions()
         details = [self.handle(question) for question in questions]
@@ -259,4 +265,5 @@ def run(self):
         ]
         details += [self.handle(question) for question in failed_questions]
         details = [detail for detail in details if detail]
+        self.save()
         return details