From 28e8c57359dccd285147c30684402a65656043c5 Mon Sep 17 00:00:00 2001 From: Sean Prashad <13009507+SeanPrashad@users.noreply.github.com> Date: Wed, 10 Aug 2022 21:46:06 -0400 Subject: [PATCH 1/2] Refactor update_questions into methods --- cron/update_questions.py | 147 +++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 53 deletions(-) diff --git a/cron/update_questions.py b/cron/update_questions.py index 6c9dd5c7..b82f4d78 100644 --- a/cron/update_questions.py +++ b/cron/update_questions.py @@ -3,37 +3,25 @@ import leetcode import leetcode.auth from datetime import datetime +from leetcode.rest import ApiException -LEETCODE_SESSION_TOKEN = os.environ.get("LEETCODE_SESSION_TOKEN") -questions_file = os.getcwd() + "/src/data/questions.json" +def create_leetcode_api(): + LEETCODE_SESSION_TOKEN = os.environ.get("LEETCODE_SESSION_TOKEN") + csrf_token = leetcode.auth.get_csrf_cookie(LEETCODE_SESSION_TOKEN) -print("=== Reading questions file ===") + configuration = leetcode.Configuration() -try: - with open(questions_file, "r") as file: - questions = json.load(file) -except Exception as e: - print(e) - exit() + configuration.api_key["x-csrftoken"] = csrf_token + configuration.api_key["csrftoken"] = csrf_token + configuration.api_key["LEETCODE_SESSION"] = LEETCODE_SESSION_TOKEN + configuration.api_key["Referer"] = "https://leetcode.com" + configuration.debug = False -print("=== Updating question metadata ===") + return leetcode.DefaultApi(leetcode.ApiClient(configuration)) -startTime = datetime.now() -csrf_token = leetcode.auth.get_csrf_cookie(LEETCODE_SESSION_TOKEN) - -configuration = leetcode.Configuration() - -configuration.api_key["x-csrftoken"] = csrf_token -configuration.api_key["csrftoken"] = csrf_token -configuration.api_key["LEETCODE_SESSION"] = LEETCODE_SESSION_TOKEN -configuration.api_key["Referer"] = "https://leetcode.com" -configuration.debug = False - -api_instance = leetcode.DefaultApi(leetcode.ApiClient(configuration)) - -for question in questions["data"]: +def get_question_metadata(api, title_slug): graphql_request = leetcode.GraphqlQuery( query='''query questionData($titleSlug: String!) { question(titleSlug: $titleSlug) { @@ -45,45 +33,98 @@ } ''', variables=leetcode.GraphqlQueryGetQuestionDetailVariables( - title_slug=question["slug"]) + title_slug=title_slug) ) - response = api_instance.graphql_post(body=graphql_request).to_dict() + try: + response = api.graphql_post(body=graphql_request) + except ApiException as e: + print( + f'Exception occurred when contacting the Leetcode GraphQL API: ${e}') + exit() - leetcode_title = response["data"]["question"]["title"] - leetcode_difficulty = response["data"]["question"]["difficulty"] - leetcode_companyTags = json.loads( - response["data"]["question"]["company_tag_stats"]) - leetcode_premium = response["data"]["question"]["is_paid_only"] + return response - # Retrieve companies who have asked this question within 0-1 year - leetcode_companies = leetcode_companyTags["1"] + leetcode_companyTags["2"] +def construct_company_tag_list(company_tags_json, sections): companies = [] - for leetcode_company in leetcode_companies: - companies.append({ - "name": leetcode_company["name"], - "slug": leetcode_company["slug"], - "frequency": leetcode_company["timesEncountered"] - }) + for section in sections: + for company in company_tags_json[section]: + companies.append({ + "name": company["name"], + "slug": company["slug"], + "frequency": company["timesEncountered"] + }) + + return sorted(companies, key=lambda d: d['frequency'], reverse=True) - companies = sorted(companies, key = lambda d: d['frequency'], reverse=True) - question["title"] = leetcode_title - question["difficulty"] = leetcode_difficulty +def update_question_metadata(question, title, difficulty, companies, is_premium): + question["title"] = title + question["difficulty"] = difficulty question["companies"] = companies - question["premium"] = leetcode_premium + question["premium"] = is_premium + + +def read_questions(file_name): + print("=== Reading questions file ===") + + try: + with open(file_name, "r") as file: + questions = json.load(file) + return questions["data"] + except Exception as e: + print( + f'Exception occurred when reading questions.json: ${e}') + exit() + + +def write_questions(file_name, data): + try: + with open(file_name, "w") as file: + data["updated"] = str(datetime.now().isoformat()) + json.dump(data, file, indent=2) + except Exception as e: + print( + f'Exception occurred when writing questions.json: ${e}') + exit() + + print("=== Wrote questions file ===") + + +def main(file_name): + api = create_leetcode_api() + questions = read_questions(file_name) + + for question in questions: + title_slug = question["slug"] + + response = get_question_metadata(api, title_slug) + + question_title = response.data.question.title + question_difficulty = response.data.question.difficulty + question_company_tags = json.loads( + response.data.question.company_tag_stats) + question_is_premium = response.data.question.is_paid_only + + # Retrieve companies who have asked this question within the following two + # company_tag_stat sections: + # 1. 0-6 months + # 2. 6 months to 1 year + companies = construct_company_tag_list( + question_company_tags, ["1", "2"]) + + update_question_metadata(question, question_title, question_difficulty, + companies, question_is_premium) + + write_questions(file_name, questions) + -print("=== Finished checking all questions ===") +if __name__ == "__main__": + file_name = os.getcwd() + "/src/data/questions.json" + startTime = datetime.now() -try: - with open(questions_file, "w") as file: - questions["updated"] = str(datetime.now().isoformat()) - json.dump(questions, file, indent=2) -except Exception as e: - print(e) - exit() + main(file_name) -print("=== Wrote questions file ===") -print(f'=== Script took: {datetime.now() - startTime} seconds ===') + print(f'=== Script took: {datetime.now() - startTime} seconds ===') From 2d8dc261de553adfd52997d76ac14e08aea7b321 Mon Sep 17 00:00:00 2001 From: Sean Prashad <13009507+SeanPrashad@users.noreply.github.com> Date: Wed, 10 Aug 2022 22:01:45 -0400 Subject: [PATCH 2/2] Improve script logging --- cron/update_questions.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cron/update_questions.py b/cron/update_questions.py index b82f4d78..8c05222b 100644 --- a/cron/update_questions.py +++ b/cron/update_questions.py @@ -61,6 +61,8 @@ def construct_company_tag_list(company_tags_json, sections): def update_question_metadata(question, title, difficulty, companies, is_premium): + print(f"🔄 Updating question metadata for {title}") + question["title"] = title question["difficulty"] = difficulty question["companies"] = companies @@ -68,36 +70,38 @@ def update_question_metadata(question, title, difficulty, companies, is_premium) def read_questions(file_name): - print("=== Reading questions file ===") + print(f"💾 Loading {file_name}") try: with open(file_name, "r") as file: questions = json.load(file) - return questions["data"] + print(f"✅ Finished loading {file_name}") + return questions except Exception as e: print( - f'Exception occurred when reading questions.json: ${e}') + f"❌ Exception occurred when reading {file_name}: {e}") exit() -def write_questions(file_name, data): +def write_questions(file_name, questions): + print(f"💾 Updating {file_name}") + try: with open(file_name, "w") as file: - data["updated"] = str(datetime.now().isoformat()) - json.dump(data, file, indent=2) + questions["updated"] = str(datetime.now().isoformat()) + json.dump(questions, file, indent=2) + print(f"✅ Finished updating {file_name}") except Exception as e: print( - f'Exception occurred when writing questions.json: ${e}') + f"❌ Exception occurred when writing {file_name}: {e}") exit() - print("=== Wrote questions file ===") - def main(file_name): api = create_leetcode_api() questions = read_questions(file_name) - for question in questions: + for question in questions["data"]: title_slug = question["slug"] response = get_question_metadata(api, title_slug) @@ -127,4 +131,4 @@ def main(file_name): main(file_name) - print(f'=== Script took: {datetime.now() - startTime} seconds ===') + print(f"⏱️ Data updated in {datetime.now() - startTime} seconds")