diff --git a/.github/workflows/sync-missing-youtube-transcripts.yml b/.github/workflows/sync-missing-youtube-transcripts.yml index 63938b21..4115e1c6 100644 --- a/.github/workflows/sync-missing-youtube-transcripts.yml +++ b/.github/workflows/sync-missing-youtube-transcripts.yml @@ -11,6 +11,11 @@ on: options: - production - dev + batch_size: + description: "Batch size (number of videos to process in this run)" + required: true + default: "50" + type: string jobs: sync-transcripts: @@ -39,3 +44,4 @@ jobs: SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} SANITY_STUDIO_DATASET: ${{ inputs.dataset }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + SYNC_BATCH_SIZE: ${{ inputs.batch_size }} diff --git a/apps/sanity/functions/sync-missing-youtube-transcripts/index.ts b/apps/sanity/functions/sync-missing-youtube-transcripts/index.ts index 7b40124a..465020b1 100644 --- a/apps/sanity/functions/sync-missing-youtube-transcripts/index.ts +++ b/apps/sanity/functions/sync-missing-youtube-transcripts/index.ts @@ -259,10 +259,27 @@ export const handler = scheduledEventHandler(async ({ context }) => { useCdn: false, }); + const batchSize = parseInt(process.env.SYNC_BATCH_SIZE || "50", 10); + console.log( `[Sync Missing YouTube Transcripts] Querying documents with YouTube URLs missing transcripts in project ${projectId}, dataset ${dataset}...`, ); + const totalMissing = await client.fetch( + `count(*[_type in ["post", "podcast", "course", "lesson", "short"] && (defined(youtube) || defined(listenLinks.youtube)) && !defined(transcript) && !(_id in path("drafts.**"))])`, + ); + + console.log( + `[Sync Missing YouTube Transcripts] Total documents currently missing transcripts in ${dataset}: ${totalMissing}`, + ); + + if (totalMissing === 0) { + console.log( + "[Sync Missing YouTube Transcripts] All videos currently have linked transcripts. Done.", + ); + return; + } + const missingDocs = await client.fetch< Array<{ _id: string; @@ -271,25 +288,19 @@ export const handler = scheduledEventHandler(async ({ context }) => { youtube: string; }> >( - `*[_type in ["post", "podcast"] && defined(youtube) && !defined(transcript) && !(_id in path("drafts.**"))][0...25]{ + `*[_type in ["post", "podcast", "course", "lesson", "short"] && (defined(youtube) || defined(listenLinks.youtube)) && !defined(transcript) && !(_id in path("drafts.**"))][0...$batchSize]{ _id, _type, title, - youtube + "youtube": coalesce(youtube, listenLinks.youtube) }`, + { batchSize }, ); console.log( - `[Sync Missing YouTube Transcripts] Found ${missingDocs.length} documents missing transcripts.`, + `[Sync Missing YouTube Transcripts] Processing batch of ${missingDocs.length} documents (batch limit: ${batchSize})...`, ); - if (missingDocs.length === 0) { - console.log( - "[Sync Missing YouTube Transcripts] All videos currently have linked transcripts. Done.", - ); - return; - } - let successCount = 0; let failCount = 0;