chromium: Secure URLs Crashing Chromium in Local SAM Testing

We are getting the error “Navigation failed because browser has disconnected!” for all secure (https) URLs. Non-secure (http) URLs are working fine.

We are struggling to troubleshoot this issue since it used to work and it’s working fine when deployed to Lambda. We haven’t made any changes to libraries. Here is our minimum test and failure:

template.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Resources:
  MyFunction:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: src/
      Handler: index.handler
      Runtime: nodejs16.x
      Architectures:
        - x86_64
      MemorySize: 1536
      Timeout: 120

package.json

{
  "name": "headless-browser-capture-url",
  "version": "1.0.0",
  "type": "module",
  "main": "index",
  "dependencies": {
    "aws-sdk": "^2.799.0",
    "@sparticuz/chromium": "^110.0.1",
    "puppeteer-core": "^19.6.0",
    "puppeteer-extra": "^3.3.6",
    "puppeteer-extra-plugin-recaptcha": "^3.6.8",
    "puppeteer-extra-plugin-stealth": "^2.11.2",
    "request": "^2.88.2"
  }
}

index.js

import chromium from '@sparticuz/chromium';
import puppeteer from 'puppeteer-core'

export async function handler(event, context, callback) {
    let args = chromium.args;
    args.push('--disable-gpu');
    const browser = await puppeteer.launch({
        args: args,
        defaultViewport: chromium.defaultViewport,
        executablePath: await chromium.executablePath(),
        headless: 'old',
    });

    const page = await browser.newPage();

    await page.goto('http://example.com');
    const pageTitle1 = await page.title();
    console.log('Insecure: '+pageTitle1);

    await page.goto('https://example.com');
    const pageTitle2 = await page.title();
    console.log('Secure: '+pageTitle2);

    await browser.close();
}

Local Test

bash-3.2$ sam local invoke MyFunction
Invoking index.handler (nodejs16.x)                                                                                                                                        
Local image is up-to-date                                                                                                                                                  
Using local image: public.ecr.aws/lambda/nodejs:16-rapid-x86_64.                                                                                                           
                                                                                                                                                                           
Mounting /headless-browser-capture-url/.aws-sam/build/MyFunction as /var/task:ro,delegated, inside runtime container  
START RequestId: 27521677-5a6b-4579-ab29-85a0ffe329c8 Version: $LATEST
2023-12-07T14:14:09.942Z        92ed4fde-d77b-44b0-8b87-d00613e902a0    INFO    Insecure: Example Domain
2023-12-07T14:14:09.991Z        92ed4fde-d77b-44b0-8b87-d00613e902a0    ERROR   Invoke Error    {"errorType":"Error","errorMessage":"Navigation failed because browser has disconnected!","stack":["Error: Navigation failed because browser has disconnected!","    at new LifecycleWatcher (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/LifecycleWatcher.js:80:192)","    at Frame.goto (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/Frame.js:180:25)","    at CDPPage.goto (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/Page.js:435:91)","    at Runtime.handler (file:///var/task/index.js:20:16)","    at processTicksAndRejections (node:internal/process/task_queues:96:5)"]}
END RequestId: 92ed4fde-d77b-44b0-8b87-d00613e902a0
REPORT RequestId: 92ed4fde-d77b-44b0-8b87-d00613e902a0  Init Duration: 0.03 ms  Duration: 1688.64 ms    Billed Duration: 1689 ms        Memory Size: 1536 MB    Max Memory Used: 1536 MB
{"errorType": "Error", "errorMessage": "Navigation failed because browser has disconnected!", "trace": ["Error: Navigation failed because browser has disconnected!", "    at new LifecycleWatcher (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/LifecycleWatcher.js:80:192)", "    at Frame.goto (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/Frame.js:180:25)", "    at CDPPage.goto (file:///var/task/node_modules/puppeteer-core/lib/esm/puppeteer/common/Page.js:435:91)", "    at Runtime.handler (file:///var/task/index.js:20:16)", "    at processTicksAndRejections (node:internal/process/task_queues:96:5)"]}
bash-3.2$