diff --git a/src/features/canvas/services/canvasAssignmentGroupService.ts b/src/features/canvas/services/canvasAssignmentGroupService.ts index 34fd0a6..58c1085 100644 --- a/src/features/canvas/services/canvasAssignmentGroupService.ts +++ b/src/features/canvas/services/canvasAssignmentGroupService.ts @@ -1,7 +1,7 @@ import { canvasApi, paginatedRequest } from "./canvasServiceUtils"; import { CanvasAssignmentGroup } from "@/features/canvas/models/assignments/canvasAssignmentGroup"; import { LocalAssignmentGroup } from "@/features/local/assignments/models/localAssignmentGroup"; -import { rateLimitAwareDelete } from "./canvasWebRequestor"; +import { rateLimitAwareDelete, rateLimitAwarePost } from "./canvasWebRequestUtils"; import { axiosClient } from "@/services/axiosUtils"; export const canvasAssignmentGroupService = { @@ -26,7 +26,7 @@ export const canvasAssignmentGroupService = { }; const { data: canvasAssignmentGroup } = - await axiosClient.post(url, body); + await rateLimitAwarePost(url, body); return { ...localAssignmentGroup, diff --git a/src/features/canvas/services/canvasAssignmentService.ts b/src/features/canvas/services/canvasAssignmentService.ts index fb7287a..20a3e10 100644 --- a/src/features/canvas/services/canvasAssignmentService.ts +++ b/src/features/canvas/services/canvasAssignmentService.ts @@ -8,6 +8,7 @@ import { getRubricCriterion } from "./canvasRubricUtils"; import { LocalCourseSettings } from "@/features/local/course/localCourseSettings"; import { axiosClient } from "@/services/axiosUtils"; import { markdownToHTMLSafe } from "@/services/htmlMarkdownUtils"; +import { rateLimitAwarePost } from "./canvasWebRequestUtils"; export const canvasAssignmentService = { async getAll(courseId: number): Promise { @@ -60,7 +61,7 @@ export const canvasAssignmentService = { }, }; - const response = await axiosClient.post(url, body); + const response = await rateLimitAwarePost(url, body); const canvasAssignment = response.data; await createRubric(canvasCourseId, canvasAssignment.id, localAssignment); @@ -152,7 +153,7 @@ const createRubric = async ( }; const rubricUrl = `${canvasApi}/courses/${courseId}/rubrics`; - const rubricResponse = await axiosClient.post( + const rubricResponse = await rateLimitAwarePost( rubricUrl, rubricBody ); diff --git a/src/features/canvas/services/canvasModuleService.ts b/src/features/canvas/services/canvasModuleService.ts index 62c9e69..e05c004 100644 --- a/src/features/canvas/services/canvasModuleService.ts +++ b/src/features/canvas/services/canvasModuleService.ts @@ -3,6 +3,7 @@ import { CanvasPage } from "@/features/canvas/models/pages/canvasPageModel"; import { canvasApi, paginatedRequest } from "./canvasServiceUtils"; import { CanvasModule } from "@/features/canvas/models/modules/canvasModule"; import { axiosClient } from "@/services/axiosUtils"; +import { rateLimitAwarePost } from "./canvasWebRequestUtils"; export const canvasModuleService = { async updateModuleItem( @@ -37,7 +38,7 @@ export const canvasModuleService = { console.log(`Creating new module item ${title}`); const url = `${canvasApi}/courses/${canvasCourseId}/modules/${canvasModuleId}/items`; const body = { module_item: { title, type, content_id: contentId } }; - await axiosClient.post(url, body); + await rateLimitAwarePost(url, body); }, async createPageModuleItem( @@ -51,7 +52,7 @@ export const canvasModuleService = { const body = { module_item: { title, type: "Page", page_url: canvasPage.url }, }; - await axiosClient.post(url, body); + await rateLimitAwarePost(url, body); }, async getCourseModules(canvasCourseId: number) { @@ -67,7 +68,7 @@ export const canvasModuleService = { name: moduleName, }, }; - const response = await axiosClient.post(url, body); + const response = await rateLimitAwarePost(url, body); return response.data.id; }, diff --git a/src/features/canvas/services/canvasPageService.ts b/src/features/canvas/services/canvasPageService.ts index fd446b8..e88bf56 100644 --- a/src/features/canvas/services/canvasPageService.ts +++ b/src/features/canvas/services/canvasPageService.ts @@ -1,7 +1,7 @@ import { CanvasPage } from "@/features/canvas/models/pages/canvasPageModel"; import { LocalCoursePage } from "@/features/local/pages/localCoursePageModels"; import { canvasApi, paginatedRequest } from "./canvasServiceUtils"; -import { rateLimitAwareDelete } from "./canvasWebRequestor"; +import { rateLimitAwareDelete, rateLimitAwarePost } from "./canvasWebRequestUtils"; import { LocalCourseSettings } from "@/features/local/course/localCourseSettings"; import { axiosClient } from "@/services/axiosUtils"; import { markdownToHTMLSafe } from "@/services/htmlMarkdownUtils"; @@ -41,7 +41,7 @@ export const canvasPageService = { }, }; - const { data: canvasPage } = await axiosClient.post(url, body); + const { data: canvasPage } = await rateLimitAwarePost(url, body); if (!canvasPage) { throw new Error("Created canvas course page was null"); } diff --git a/src/features/canvas/services/canvasQuizService.ts b/src/features/canvas/services/canvasQuizService.ts index a04db64..6f79eb2 100644 --- a/src/features/canvas/services/canvasQuizService.ts +++ b/src/features/canvas/services/canvasQuizService.ts @@ -12,6 +12,7 @@ import { LocalCourseSettings } from "@/features/local/course/localCourseSettings import { axiosClient } from "@/services/axiosUtils"; import { markdownToHTMLSafe } from "@/services/htmlMarkdownUtils"; import { escapeMatchingText } from "@/services/utils/questionHtmlUtils"; +import { rateLimitAwareDelete, rateLimitAwarePost } from "./canvasWebRequestUtils"; export const getAnswers = ( question: LocalQuizQuestion, @@ -64,7 +65,7 @@ const createQuestionOnly = async ( }, }; - const response = await axiosClient.post(url, body); + const response = await rateLimitAwarePost(url, body); const newQuestion = response.data; if (!newQuestion) throw new Error("Created question is null"); @@ -86,7 +87,7 @@ const hackFixQuestionOrdering = async ( })); const url = `${canvasApi}/courses/${canvasCourseId}/quizzes/${canvasQuizId}/reorder`; - await axiosClient.post(url, { order }); + await rateLimitAwarePost(url, { order }); }; const verifyQuestionOrder = async ( @@ -95,7 +96,7 @@ const verifyQuestionOrder = async ( localQuiz: LocalQuiz ): Promise => { console.log("Verifying question order in Canvas quiz"); - + try { const canvasQuestions = await canvasQuizService.getQuizQuestions( canvasCourseId, @@ -113,19 +114,21 @@ const verifyQuestionOrder = async ( // Verify that questions are in the correct order by comparing text content // We'll use a simple approach: strip HTML tags and compare the core text content const stripHtml = (html: string): string => { - return html.replace(/<[^>]*>/g, '').trim(); + return html.replace(/<[^>]*>/g, "").trim(); }; for (let i = 0; i < localQuiz.questions.length; i++) { const localQuestion = localQuiz.questions[i]; const canvasQuestion = canvasQuestions[i]; - + const localQuestionText = localQuestion.text.trim(); const canvasQuestionText = stripHtml(canvasQuestion.question_text).trim(); // Check if the question text content matches (allowing for HTML conversion differences) - if (!canvasQuestionText.includes(localQuestionText) && - !localQuestionText.includes(canvasQuestionText)) { + if ( + !canvasQuestionText.includes(localQuestionText) && + !localQuestionText.includes(canvasQuestionText) + ) { console.error( `Question order mismatch at position ${i}:`, `Local: "${localQuestionText}"`, @@ -135,9 +138,14 @@ const verifyQuestionOrder = async ( } // Verify position is correct - if (canvasQuestion.position !== undefined && canvasQuestion.position !== i + 1) { + if ( + canvasQuestion.position !== undefined && + canvasQuestion.position !== i + 1 + ) { console.error( - `Question position mismatch at index ${i}: Canvas position is ${canvasQuestion.position}, expected ${i + 1}` + `Question position mismatch at index ${i}: Canvas position is ${ + canvasQuestion.position + }, expected ${i + 1}` ); return false; } @@ -294,7 +302,10 @@ export const canvasQuizService = { }, }; - const { data: canvasQuiz } = await axiosClient.post(url, body); + const { data: canvasQuiz } = await rateLimitAwarePost( + url, + body + ); await createQuizQuestions( canvasCourseId, canvasQuiz.id, @@ -305,6 +316,6 @@ export const canvasQuizService = { }, async delete(canvasCourseId: number, canvasQuizId: number) { const url = `${canvasApi}/courses/${canvasCourseId}/quizzes/${canvasQuizId}`; - await axiosClient.delete(url); + await rateLimitAwareDelete(url); }, }; diff --git a/src/features/canvas/services/canvasWebRequestor.ts b/src/features/canvas/services/canvasWebRequestUtils.ts similarity index 68% rename from src/features/canvas/services/canvasWebRequestor.ts rename to src/features/canvas/services/canvasWebRequestUtils.ts index 2c496c7..89ab36e 100644 --- a/src/features/canvas/services/canvasWebRequestor.ts +++ b/src/features/canvas/services/canvasWebRequestUtils.ts @@ -1,5 +1,5 @@ import { axiosClient } from "@/services/axiosUtils"; -import { AxiosResponse } from "axios"; +import { AxiosResponse, AxiosRequestConfig } from "axios"; const rateLimitRetryCount = 6; const rateLimitSleepInterval = 1000; @@ -16,21 +16,26 @@ export const isRateLimited = async ( ); }; -// const rateLimitAwarePost = async (url: string, body: unknown, retryCount = 0) => { -// const response = await axiosClient.post(url, body); +export const rateLimitAwarePost = async ( + url: string, + body: unknown, + config?: AxiosRequestConfig, + retryCount = 0 +): Promise> => { + const response = await axiosClient.post(url, body, config); -// if (await isRateLimited(response)) { -// if (retryCount < rateLimitRetryCount) { -// console.info( -// `Hit rate limit on post, retry count is ${retryCount} / ${rateLimitRetryCount}, retrying` -// ); -// await sleep(rateLimitSleepInterval); -// return await rateLimitAwarePost(url, body, retryCount + 1); -// } -// } + if (await isRateLimited(response)) { + if (retryCount < rateLimitRetryCount) { + console.info( + `Hit rate limit on post, retry count is ${retryCount} / ${rateLimitRetryCount}, retrying` + ); + await sleep(rateLimitSleepInterval); + return await rateLimitAwarePost(url, body, config, retryCount + 1); + } + } -// return response; -// }; + return response; +}; export const rateLimitAwareDelete = async ( url: string, diff --git a/src/features/canvas/services/files/canvasFileService.ts b/src/features/canvas/services/files/canvasFileService.ts index 2647dd6..c612ebf 100644 --- a/src/features/canvas/services/files/canvasFileService.ts +++ b/src/features/canvas/services/files/canvasFileService.ts @@ -4,10 +4,11 @@ import axios from "axios"; import { canvasApi } from "../canvasServiceUtils"; import { axiosClient } from "@/services/axiosUtils"; import FormData from "form-data"; +import { rateLimitAwarePost } from "../canvasWebRequestUtils"; export const downloadUrlToTempDirectory = async ( sourceUrl: string -): Promise<{fileName: string, success: boolean}> => { +): Promise<{ fileName: string; success: boolean }> => { try { const fileName = path.basename(new URL(sourceUrl).pathname) || `tempfile-${Date.now()}`; @@ -16,10 +17,10 @@ export const downloadUrlToTempDirectory = async ( responseType: "arraybuffer", }); await fs.writeFile(tempFilePath, response.data); - return {fileName: tempFilePath, success: true}; + return { fileName: tempFilePath, success: true }; } catch (error) { console.log("Error downloading or saving the file:", sourceUrl, error); - return {fileName: sourceUrl, success: false}; + return { fileName: sourceUrl, success: false }; } }; @@ -45,7 +46,10 @@ export const uploadToCanvasPart1 = async ( formData.append("name", path.basename(pathToUpload)); formData.append("size", (await getFileSize(pathToUpload)).toString()); - const response = await axiosClient.post(url, formData); + const response = await rateLimitAwarePost<{ + upload_url: string; + upload_params: string; + }>(url, formData); const upload_url = response.data.upload_url; const upload_params = response.data.upload_params; @@ -77,10 +81,14 @@ export const uploadToCanvasPart2 = async ({ const fileName = path.basename(pathToUpload); formData.append("file", fileBuffer, fileName); - const response = await axiosClient.post(upload_url, formData, { - headers: formData.getHeaders(), - validateStatus: (status) => status < 500, - }); + const response = await rateLimitAwarePost<{ url: string }>( + upload_url, + formData, + { + headers: formData.getHeaders(), + validateStatus: (status) => status < 500, + } + ); if (response.status === 301) { const redirectUrl = response.headers.location;