From 2167fef4b9e28765a18bd06aeb1e86ba0eb7998e Mon Sep 17 00:00:00 2001
From: Sean Prashad <13009507+SeanPrashad@users.noreply.github.com>
Date: Wed, 10 Aug 2022 22:55:50 -0400
Subject: [PATCH 1/2] Introduce multithreading for requests
Fixes #217
---
cron/update_questions.py | 63 +++++++++++++++++++++++-----------------
src/data/questions.json | 2 +-
2 files changed, 38 insertions(+), 27 deletions(-)
diff --git a/cron/update_questions.py b/cron/update_questions.py
index 8c05222b..02135316 100644
--- a/cron/update_questions.py
+++ b/cron/update_questions.py
@@ -1,9 +1,11 @@
import os
import json
+import uuid
import leetcode
import leetcode.auth
from datetime import datetime
from leetcode.rest import ApiException
+from concurrent.futures import ThreadPoolExecutor, as_completed
def create_leetcode_api():
@@ -26,6 +28,7 @@ def get_question_metadata(api, title_slug):
query='''query questionData($titleSlug: String!) {
question(titleSlug: $titleSlug) {
title
+ titleSlug
difficulty
companyTagStats
isPaidOnly
@@ -38,13 +41,12 @@ def get_question_metadata(api, title_slug):
try:
response = api.graphql_post(body=graphql_request)
+ return response
except ApiException as e:
print(
f'Exception occurred when contacting the Leetcode GraphQL API: ${e}')
exit()
- return response
-
def construct_company_tag_list(company_tags_json, sections):
companies = []
@@ -60,13 +62,24 @@ def construct_company_tag_list(company_tags_json, sections):
return sorted(companies, key=lambda d: d['frequency'], reverse=True)
-def update_question_metadata(question, title, difficulty, companies, is_premium):
- print(f"🔄 Updating question metadata for {title}")
+def update_question_metadata(question, response):
+ question_title = response.data.question.title
+ question_difficulty = response.data.question.difficulty
+ question_company_tags = json.loads(
+ response.data.question.company_tag_stats)
+ question_is_premium = response.data.question.is_paid_only
+
+ # Retrieve companies who have asked this question within the following two
+ # company_tag_stat sections:
+ # 1. 0-6 months
+ # 2. 6 months to 1 year
+ companies = construct_company_tag_list(
+ question_company_tags, ["1", "2"])
- question["title"] = title
- question["difficulty"] = difficulty
+ question["title"] = question_title
+ question["difficulty"] = question_difficulty
question["companies"] = companies
- question["premium"] = is_premium
+ question["premium"] = question_is_premium
def read_questions(file_name):
@@ -97,30 +110,28 @@ def write_questions(file_name, questions):
exit()
-def main(file_name):
- api = create_leetcode_api()
- questions = read_questions(file_name)
+def runners(api, question_list):
+ print(f"📡 Retrieving question metadata from Leetcode")
- for question in questions["data"]:
- title_slug = question["slug"]
+ threads = []
- response = get_question_metadata(api, title_slug)
+ with ThreadPoolExecutor(max_workers=3) as executor:
+ for question in question_list:
+ title_slug = question["slug"]
+ threads.append(executor.submit(
+ get_question_metadata, api, title_slug))
- question_title = response.data.question.title
- question_difficulty = response.data.question.difficulty
- question_company_tags = json.loads(
- response.data.question.company_tag_stats)
- question_is_premium = response.data.question.is_paid_only
+ for task in as_completed(threads):
+ update_question_metadata(question, task.result())
- # Retrieve companies who have asked this question within the following two
- # company_tag_stat sections:
- # 1. 0-6 months
- # 2. 6 months to 1 year
- companies = construct_company_tag_list(
- question_company_tags, ["1", "2"])
+ print(f"✅ Finished retrieving question metadata from Leetcode")
+
+
+def main(file_name):
+ api = create_leetcode_api()
+ questions = read_questions(file_name)
- update_question_metadata(question, question_title, question_difficulty,
- companies, question_is_premium)
+ runners(api, questions["data"])
write_questions(file_name, questions)
diff --git a/src/data/questions.json b/src/data/questions.json
index 7ebb9eb1..9071ed00 100644
--- a/src/data/questions.json
+++ b/src/data/questions.json
@@ -1,5 +1,5 @@
{
- "updated": "2022-08-07T17:24:54.842630",
+ "updated": "2022-08-10T23:48:26.115555",
"data": [
{
"id": 0,
From 229113176520ed57b3bc256c6e33b6d271cacc39 Mon Sep 17 00:00:00 2001
From: Sean Prashad <13009507+SeanPrashad@users.noreply.github.com>
Date: Wed, 10 Aug 2022 23:50:08 -0400
Subject: [PATCH 2/2] Final testing
---
src/data/questions.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/data/questions.json b/src/data/questions.json
index 9071ed00..06a21259 100644
--- a/src/data/questions.json
+++ b/src/data/questions.json
@@ -1,5 +1,5 @@
{
- "updated": "2022-08-10T23:48:26.115555",
+ "updated": "2022-08-10T23:49:33.616820",
"data": [
{
"id": 0,