[ { "updatedAt": "2025-11-15T07:17:59.000Z", "createdAt": "2025-10-31T02:57:38.050Z", "id": "cytCUDs1P7434yDT", "name": "客戶B - Firecrawl", "description": null, "active": false, "isArchived": false, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ -80, 144 ], "id": "4ab49b1e-3aec-4704-b025-e960e8a046be", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "Firecrawl", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 80, 144 ], "id": "250b7264-4951-48ce-ba42-5010d743478f", "name": "Get row(s) in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "batchSize": 5, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 400, 144 ], "id": "cf8e01dd-7f85-468b-b4b3-1beaa30f259e", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"pricing/.*\",\n \"products/.*\",\n \"services/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"home/.*\",\n \"about/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customer-stories/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 576, 208 ], "id": "53bf31e4-a49d-4fda-abcd-e90d6c11a9b1", "name": "Crawl", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 896, 208 ], "id": "4d391233-16d1-42c7-b85e-a8c0a8dd6683", "name": "Get Crawl Dataset", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1056, 208 ], "id": "a14a7b92-f99b-4fec-baee-4c079d9c5cd6", "name": "If" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ 240, 144 ], "id": "f7360b92-2cea-4794-aeb4-fd870e2a6c17", "name": "Filter" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "reasoningEffort": "low", "timeout": 60000, "maxRetries": 3 } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 816, 656 ], "id": "ba388003-59ee-4f0c-bbce-94799f0a8a01", "name": "OpenAI Chat Model", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "jsonSchemaExample": "{\n\t\"Case Study Company\": \"Porkbun\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 960, 656 ], "id": "dff1fd15-fcbb-4e73-bf8d-ca6dcc0ecf68", "name": "Structured Output Parser" }, { "parameters": { "content": "## Step 1: Crawl websites using firecrawl\n", "height": 304, "width": 1888, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ -144, 48 ], "typeVersion": 1, "id": "001cc7d5-fc4e-4d0a-8f73-a15098eef8bd", "name": "Sticky Note" }, { "parameters": { "content": "## Step 2: Extract data from firecrawl output\n ", "height": 448, "width": 1376 }, "type": "n8n-nodes-base.stickyNote", "position": [ 16, 352 ], "typeVersion": 1, "id": "885c05d0-1bd2-451d-9830-57da3ea7220f", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 448, "width": 352, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ 1392, 352 ], "typeVersion": 1, "id": "b875fcc5-22c1-43aa-b1e2-ccd17b197aba", "name": "Sticky Note2" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "Final Sheet", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $json.output[\"Job Title\"] }}", "company": "={{ $json.output[\"Company Name\"] }}", "case study": "={{ $('Case Study').item.json.output[\"Case Study Company\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "case study", "displayName": "case study", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 1504, 480 ], "id": "392e2783-0df9-4608-894c-cf140e522dcd", "name": "Append or update row in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "reasoningEffort": "low", "timeout": 60000, "maxRetries": 3 } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1120, 656 ], "id": "d00b7bf6-a1ed-487c-9c53-d1c14bdb535e", "name": "OpenAI Chat Model1", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "jsonSchemaExample": "{\n \"Company Name\": \"客戶B\",\n \"Job Title\": \"CEO\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1264, 656 ], "id": "71698709-5875-4cae-9fb4-f5445cecadb6", "name": "Structured Output Parser1" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $('Merge Markdown').item.json.merged_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are a precise data extractor for client brand information from crawled web page markdown. Your task is to process the provided crawled page markdown and extract exactly ONE client brand name, outputting it in JSON format inside an \"output\" key.\n\n\"Case Study Company\": Extract exactly ONE client brand name from the crawled markdown.\n- Scope: Search ONLY in the provided markdown.\n- Prefer sections like: \"Our Work\", \"Case Studies\", \"Work\", \"Projects\", \"Portfolio\", \"Results\", \"Clients\", \"Testimonials\", \"Success Stories\", \"Reviews\".\n- What counts as a client brand: A named client in a case study, project, result, or testimonial attribution. Accept names in titles, headings, captions, testimonial attributions.\n- Strict rules:\n - Output must be PLAIN TEXT brand name ONLY. No brackets, quotes, notes, or extra words.\n - If you see a bare domain like \"goolets.com\", strip subdomains and TLDs and return \"Goolets\".\n - Remove anything after commas, dashes (– or -), colons, or prepositions such as \"for\", \"with\", \"by\", \"from\".\n - Stop before legal suffixes: Inc, LLC, Ltd, Limited, PLC, S.A., GmbH, BV, LLP, Co, Company, Corp, Corporation, Holdings, Group. Do not include these.\n - Return only the first 1–3 words that form the brand (typically the first 2 words).\n - Preserve proper capitalization; if unclear, use Title Case.\n - Do NOT return the current company's own name or its product lines.\n - If all you find are generic or anonymous labels (e.g., \"Leading SaaS Company\", \"Global Retailer\", \"Confidential\"), treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) The context explicitly indicates client work (case study/project/results/testimonial attribution or a “Clients” list clearly labeled as customers).\n B) The extracted name is a company/brand (not a product name, media outlet, award, technology partner, or the host company itself).\n C) The name appears as contiguous text that clearly maps to a client brand, not a vendor or press mention.\n D) If the source is a bare domain, the markdown must indicate this entity is the client/customer; otherwise do NOT use the domain.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by”.\n - Pure logo walls with unclear role (customer vs partner) and no case-study or client labeling.\n - Ambiguous sections where multiple brand names appear but none is clearly a client attribution.\n- If NOT FOUND or NOT CERTAIN: Return an empty string \"\". Do NOT write \"N/A\", \"None\", or any explanation.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Case Study Company\": \"extracted name or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 816, 480 ], "id": "74f9a427-3e97-44d1-bbfe-253aa828c3dd", "name": "Case Study", "alwaysOutputData": false }, { "parameters": { "promptType": "define", "text": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1120, 480 ], "id": "21712e60-7d9a-4b93-a92d-2998512b67e0", "name": "Format Names" }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 464, 480 ], "id": "048ee049-b823-4cf8-974b-0e3fdbefad73", "name": "Merge Markdown", "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "const merged = $input.item.json.merged_markdown; // 從前輸出取 merged_markdown\nconst front = merged.slice(0, 1000);\nconst back = merged.slice(-1000);\nconst shortened = front + back;\nreturn { json: { shortened_markdown: shortened } }; // 輸出 shortened_markdown" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 640, 480 ], "id": "b4298c75-8942-4ff1-858a-863b134db484", "name": "Shorten Markdown", "alwaysOutputData": true }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1408, 224 ], "id": "53ff5fda-047d-4423-8455-dacd5c7d2211", "name": "Get Crawl Dataset1", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1568, 224 ], "id": "41a1b5f9-732d-4fbc-848c-3fd51353f348", "name": "If1" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 288, 480 ], "id": "be59c6a1-ec4a-4840-a891-b2a41c502976", "name": "Get Crawl Dataset2", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 20 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1248, 224 ], "id": "46638764-4994-4433-99a7-17bb81a2dd31", "name": "20 sec", "webhookId": "a9c81d6c-e120-4e64-80a1-c105c34d6f7f" }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 736, 208 ], "id": "dc1b4286-e1af-495b-b667-f76e62d6076f", "name": "10 sec", "webhookId": "a9c81d6c-e120-4e64-80a1-c105c34d6f7f" }, { "parameters": { "amount": 60 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 112, 480 ], "id": "c80b1304-dd31-4761-bbc4-2e33dc2c84da", "name": "60 sec", "webhookId": "a9c81d6c-e120-4e64-80a1-c105c34d6f7f" } ], "connections": { "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Get row(s) in sheet", "type": "main", "index": 0 } ] ] }, "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "Crawl", "type": "main", "index": 0 } ] ] }, "Crawl": { "main": [ [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset": { "main": [ [ { "node": "If", "type": "main", "index": 0 } ] ] }, "If": { "main": [ [ { "node": "Merge Markdown", "type": "main", "index": 0 } ], [ { "node": "20 sec", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Case Study", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "Case Study", "type": "ai_outputParser", "index": 0 } ] ] }, "OpenAI Chat Model1": { "ai_languageModel": [ [ { "node": "Format Names", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser1": { "ai_outputParser": [ [ { "node": "Format Names", "type": "ai_outputParser", "index": 0 } ] ] }, "Case Study": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] }, "Format Names": { "main": [ [ { "node": "Append or update row in sheet", "type": "main", "index": 0 } ] ] }, "Merge Markdown": { "main": [ [ { "node": "Shorten Markdown", "type": "main", "index": 0 } ] ] }, "Shorten Markdown": { "main": [ [ { "node": "Case Study", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset1": { "main": [ [ { "node": "If1", "type": "main", "index": 0 } ] ] }, "If1": { "main": [ [ { "node": "Merge Markdown", "type": "main", "index": 0 } ], [ { "node": "60 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset2": { "main": [ [ { "node": "Merge Markdown", "type": "main", "index": 0 } ] ] }, "20 sec": { "main": [ [ { "node": "Get Crawl Dataset1", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get Crawl Dataset", "type": "main", "index": 0 } ] ] }, "60 sec": { "main": [ [ { "node": "Get Crawl Dataset2", "type": "main", "index": 0 } ] ] }, "Append or update row in sheet": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "4fe76ce1-bbb5-4e77-909e-660761cdb8ba", "activeVersionId": null, "versionCounter": 2, "triggerCount": 0, "tags": [ { "updatedAt": "2025-11-07T01:03:45.421Z", "createdAt": "2025-11-07T01:03:45.421Z", "id": "NmR44U7USYyu8XcS", "name": "客戶B" } ], "shared": [ { "updatedAt": "2025-10-31T02:57:38.053Z", "createdAt": "2025-10-31T02:57:38.053Z", "role": "workflow:owner", "workflowId": "cytCUDs1P7434yDT", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-15T21:47:20.000Z", "createdAt": "2025-11-02T02:30:25.596Z", "id": "81w1WIRKylyqkno3", "name": "Error Alert", "description": null, "active": false, "isArchived": false, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.errorTrigger", "typeVersion": 1, "position": [ 0, 0 ], "id": "9e090fb5-4fd0-439b-b87d-de74b599a50f", "name": "Error Trigger" }, { "parameters": { "authentication": "webhook", "content": "={{ $json.workflow.name }} Error\n\nURL: {{ $json.execution.url }}\nError Message: {{ $json.execution.error.message }}\nError Node: {{ $json.execution.lastNodeExecuted }}", "options": {} }, "type": "n8n-nodes-base.discord", "typeVersion": 2, "position": [ 176, 0 ], "id": "af9b8fb5-c4d3-411e-85d2-560ff8c4d552", "name": "Discord", "webhookId": "6631808a-a4c7-4938-86e0-c900da2b82ee", "credentials": { "discordWebhookApi": { "id": "LSdieZQeyj5CFl6s", "name": "Discord Webhook account" } } } ], "connections": { "Error Trigger": { "main": [ [ { "node": "Discord", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "d1b38085-b70f-485b-b2ff-5012282da86d", "activeVersionId": null, "versionCounter": 16, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-02T02:30:25.598Z", "createdAt": "2025-11-02T02:30:25.598Z", "role": "workflow:owner", "workflowId": "81w1WIRKylyqkno3", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-20T20:39:18.000Z", "createdAt": "2025-11-03T19:35:42.444Z", "id": "VjcOz2TJQTTjtY51", "name": "Crawl4AI - ICP + Service", "description": null, "active": false, "isArchived": true, "nodes": [ { "parameters": { "method": "POST", "url": "https://crawler.internal/crawl", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"urls\": \"{{ $json.Website }}\",\n \"crawler_params\": {\n \"headless\": true\n },\n \"extraction_config\": {\n \"type\": \"llm\",\n \"params\": {\n \"llm_config\": {\n \"provider\": \"openai/gpt5-nano\"\n },\n \"schema\": {\n \"title\": \"WebsiteExtraction\",\n \"type\": \"object\",\n \"properties\": {\n \"icp\": {\n \"title\": \"ICP\",\n \"description\": \"Short phrase describing the ideal customer profile (e.g., 'investment banks and asset managers' or 'talent acquisition teams across industries'). Less than 10 words, plural form if appropriate, focus on industries/types/roles from About/Solutions sections.\",\n \"type\": \"string\"\n },\n \"service_benefit\": {\n \"title\": \"Service Benefit\",\n \"description\": \"Short verb + noun phrase describing the core service benefit (e.g., 'analyze market data and identify opportunities' or 'accelerate talent sourcing'). 2-6 words, benefit-oriented, from Products/Features sections.\",\n \"type\": \"string\"\n },\n \"score\": {\n \"title\": \"Score\",\n \"description\": \"ICP match score from 1-10 based on principles: Target B2B Software/SaaS (high score); AVOID B2C SaaS, Advertising/Marketing/Sales Software, E-Commerce targeting companies, Managed Services Providers/IT Services/Staffing, Higher Education SaaS, Events/Planners/Conferences SaaS, Web3/Blockchain/Cryptocurrency software, Government SaaS, Salesforce partners, Medical Billing/Clinic Scheduling software. If not B2B Software/SaaS industries or has obvious low monthly pricing ($250-500/month) on website, score 3 or below.\",\n \"type\": \"integer\"\n }\n },\n \"required\": [\"icp\", \"service_benefit\", \"score\"]\n },\n \"extraction_type\": \"schema\",\n \"instruction\": \"From the crawled content, extract ICP as a concise phrase for target customers (prioritize About/Our Customers sections, avoid broad terms like 'businesses'). Extract Service Benefit as a verb-led benefit phrase (prioritize Products/Solutions/Features, ensure it fits 'looking to {service_benefit}'). Then, score the ICP match on a 1-10 scale: Target B2B Software/SaaS (high score); AVOID B2C SaaS, Advertising/Marketing/Sales Software, E-Commerce targeting companies, Managed Services Providers/IT Services/Staffing, Higher Education SaaS, Events/Planners/Conferences SaaS, Web3/Blockchain/Cryptocurrency software, Government SaaS, Salesforce partners, Medical Billing/Clinic Scheduling software. If not B2B Software/SaaS industries or has obvious low monthly pricing ($250-500/month) on website, score 3 or below. Do not guess; if unclear, leave fields empty or score low. Ensure outputs are logical and natural when combined in 'we can connect you with {icp} that are looking to {service_benefit}'.\"\n }\n },\n \"cache_mode\": \"bypass\"\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -848, -288 ], "id": "fa258544-36f2-452c-a25f-bc805278e32c", "name": "LLM Crawl Request", "credentials": { "httpHeaderAuth": { "id": "57qqazpEi0nmDEMR", "name": "Header Auth account" } } }, { "parameters": {}, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ -688, -288 ], "id": "19ef947b-724d-4607-a292-946bd0ee359a", "name": "Wait For LLM Crawl", "webhookId": "75c74b5b-96bc-41bc-a259-dec4548938a9" }, { "parameters": { "url": "=https://crawler.internal/task/{{ $json.task_id }}", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -528, -288 ], "id": "8c231d2c-9fd3-4246-840b-4de262302b59", "name": "Get LLM Crawl", "credentials": { "httpHeaderAuth": { "id": "57qqazpEi0nmDEMR", "name": "Header Auth account" } } }, { "parameters": { "mode": "raw", "jsonOutput": "={\n \"task_id\": \"{{ $('LLM Crawl Request').item.json.task_id }}\",\n \"status\": \"{{ $json.status }}\"\n}\n", "includeOtherFields": true, "options": {} }, "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ -368, -288 ], "id": "14835776-7b82-4e8e-899b-6d6a5bfd9a92", "name": "LLM TID Field" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "f07a7b5c-71e5-4aca-adce-e01b0d6c7fde", "leftValue": "={{ $json.status }}", "rightValue": "=pending", "operator": { "type": "string", "operation": "notEquals" } }, { "id": "7f622642-d24c-47de-b4be-93ce15948db3", "leftValue": "={{ $json.status }}", "rightValue": "=processing", "operator": { "type": "string", "operation": "notEquals" } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ -208, -176 ], "id": "3dcab224-26b7-4d7b-bb8c-dbc4738232c9", "name": "LLM If In Process" }, { "parameters": { "mode": "raw", "jsonOutput": "={\n \"result\": {{ $('Get LLM Crawl').item.json.result.extracted_content }}\n}", "includeOtherFields": true, "options": {} }, "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ -48, -192 ], "id": "e0864953-1b20-44a6-b690-97a1f6f3d6f2", "name": "LLM Result" }, { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n Firecrawl Input", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ -1328, -352 ], "id": "2125b067-2b2a-426d-9c05-53176137ada8", "name": "Get row(s) in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ -1168, -352 ], "id": "72f1d3b8-70fa-4e73-8db2-a5d956d63a35", "name": "Filter" }, { "parameters": { "batchSize": 2, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ -1008, -352 ], "id": "c7dea230-cd9e-40f6-9582-448971cdbdff", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "jsCode": "return items[0].json.result[0];" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 112, -192 ], "id": "c3130ae6-6b6b-4542-be6f-129d372702cc", "name": "Code in JavaScript" }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $('Filter').item.json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\",\n \"home/.*\",\n \"about/.*\",\n \"privacy/.*\",\n \"cookies/.*\",\n \"blog-preview/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customers/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"products/.*\",\n \"services/.*\",\n \"pricing/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -1488, 96 ], "id": "33b575e4-4c8d-4c72-80ed-24d561a86ebe", "name": "Crawl Case Study", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ -528, 176 ], "id": "47fb96c2-3429-4e0d-a821-c01bb1edc822", "name": "Case Study Merge", "alwaysOutputData": true }, { "parameters": {}, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ -1328, 96 ], "id": "1175e02f-8e69-4ad2-82a3-477edafceb37", "name": "Wait For LLM Crawl1", "webhookId": "75c74b5b-96bc-41bc-a259-dec4548938a9" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -1168, 96 ], "id": "9ebdd124-768e-4695-b30a-10a2a91ada33", "name": "Get LLM Crawl1", "credentials": { "httpHeaderAuth": { "id": "57qqazpEi0nmDEMR", "name": "Header Auth account" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "1b4bd57c-597b-4472-bf1d-c81c11cd1c2e", "leftValue": "={{ $('Get LLM Crawl1').item.json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ -1008, 96 ], "id": "d2eecd85-73e6-476f-86c7-1fc0df9c2609", "name": "LLM If In Process1" }, { "parameters": { "jsonSchemaExample": "{\n \"ICP Score\": \"7\",\n \"Company Name\": \"Cyberleaf\",\n \"Job Title\": \"Sales Director\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ -224, 336 ], "id": "e15ec2ec-94fe-4eff-b8b1-5fbe6d411005", "name": "Structured Output Parser3" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $('Case Study Merge').item.json.merged_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are an ICP scoring system for identifying high-ticket B2B SaaS companies ($25k+ deals). Analyze the provided crawled markdown from the company's website (focusing only on homepage and pricing page content) to determine fit. Target: B2B SaaS for businesses. AVOID: B2C SaaS, Advertising/Marketing/Sales Software, E-Commerce targeting companies (ecommerce software/marketplaces), Managed IT Services/Staffing, Higher Education SaaS, Events/Planners/Conferences SaaS, Web3/Blockchain/Cryptocurrency software, Government SaaS, Salesforce partners, Medical Billing/Clinic Scheduling software. If obviously not software/SaaS or has clear low monthly pricing ($250-500/month), score 3 or below—only after confirmation to avoid false lows.\n\nResearch Method:\n1. Analyze only homepage and pricing page content from the crawled markdown.\n2. Verify alignment: Check for B2B SaaS indicators (e.g., \"enterprise software\"). Negative if matches avoid list (e.g., \"marketing tool\", \"ecommerce platform\").\n3. Check audience: Infer from homepage/pricing for business focus. Negative if avoid categories (e.g., \"for consumers\", \"government use\").\n4. Pricing clues: Look for low plans ($250-500/month)—negative; favor custom/enterprise.\n5. If insufficient data, output empty string for score. Otherwise, assign at least 1.\n\nClassification: Rate 1-10:\n- 1–3: Not a fit (confirmed avoid category, not software/SaaS, or explicit low pricing).\n- 4–6: Moderate (some B2B but unclear/overlap).\n- 7–8: Good (strong B2B SaaS, no avoids, higher pricing hints).\n- 9–10: Perfect (clear high-ticket B2B SaaS, no negatives).\n\nNext, clean the provided original company name using the crawled markdown and company URL for confirmation only. Guidelines: Retain core name; remove legal suffixes (e.g., LLC, Inc); remove descriptors (e.g., Holdings, Group) if not core; max 3-4 words; correct casing; no additions or assumptions; prioritize original unless clear conflict.\n\nFinally, clean the provided original job title to primary role only. Guidelines: Remove duplicates/connectors (e.g., \"Founder and CEO\" → \"CEO\"); standardize (e.g., \"Chief Executive Officer\" → \"CEO\"); max 3 words; Title Case; keep highest role (prefer executive); no additions; if unclear, empty string.\n\nOutput format: Return ONLY a JSON object: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\", \"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text. If no viable value for a field, return empty string \"\"." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ -368, 176 ], "id": "0b2daa1f-a90d-4b90-a282-c6c0105845bc", "name": "ICP Scoring" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ -368, 336 ], "id": "65265c7a-7df0-4cd2-b43a-8321a71b8515", "name": "OpenAI Chat Model2", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ -848, 176 ], "id": "52a893c3-c9b0-4be9-8d12-6d43150e72d4", "name": "10 sec", "webhookId": "2765ddc6-5b45-496a-8cce-3c3be41ec5e8" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -688, 176 ], "id": "01ea6b2c-a8ab-479a-bd5b-b0b9b8ab8e87", "name": "Get LLM Crawl2", "credentials": { "httpHeaderAuth": { "id": "57qqazpEi0nmDEMR", "name": "Header Auth account" } } }, { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ -1488, -352 ], "id": "8b4477c1-0965-4de1-b640-284ca9d81171", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "014459a4-db94-4efa-a4f3-683d8d1470f9", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "7106af8f-a9a8-4a68-9062-086eb65be6ac", "leftValue": "={{ $('Code in JavaScript').item.json.icp }}", "rightValue": "", "operator": { "type": "string", "operation": "notExists", "singleValue": true } }, { "id": "f841b6f4-546c-499b-8d15-bbddc239bc04", "leftValue": "={{ $('Code in JavaScript').item.json.service_benefit }}", "rightValue": "", "operator": { "type": "string", "operation": "notExists", "singleValue": true } }, { "id": "471f3eca-8852-4b7f-b954-e5dfe2a0e5a5", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "1", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "61234ae2-4531-4de6-a7a9-7c8df8ced62c", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "2", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "734373d9-a283-4d14-aa8e-de28e9b08572", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "3", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ -80, 176 ], "id": "0a415a99-d2ad-4ad1-93f5-26f081531b44", "name": "If Score < 3" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ☆ Qualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json[\"First Name\"] }}", "last name": "={{ $('Get row(s) in sheet').item.json[\"Last Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json[\"Person Linkedin Url\"] }}", "company": "={{ $json.output['Company Name'] }}", "title": "={{ $json.output['Job Title'] }}", "icp score": "={{ $json.output['ICP Score'] }}", "icp": "={{ $('Code in JavaScript').item.json.icp }}", "service benefits": "={{ $('Code in JavaScript').item.json.service_benefit }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "icp", "displayName": "icp", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "service benefits", "displayName": "service benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "icp score", "displayName": "icp score", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 80, 336 ], "id": "88260d3f-98a6-4d60-a4bf-5faa525b6b0b", "name": "Qualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ⨯ Unqualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $json.output['Job Title'] }}", "company": "={{ $json.output['Company Name'] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "icp score": "={{ $json.output['ICP Score'] }}", "icp": "={{ $('Code in JavaScript').item.json.icp }}", "service benefits": "={{ $('Code in JavaScript').item.json.service_benefit }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "icp", "displayName": "icp", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "service benefits", "displayName": "service benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "icp score", "displayName": "icp score", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 80, 160 ], "id": "1d5dc675-3a8d-4a96-8a98-3f22d00f15ce", "name": "Unqualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": {}, "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 240, 176 ], "id": "f85093a1-1f7b-482a-92b0-5513f2327bcf", "name": "Merge" }, { "parameters": { "content": "## Step 1: Crawl websites using Crawl4AI\n", "height": 464, "width": 1968, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ -1568, -448 ], "typeVersion": 1, "id": "ec6627dc-7ddc-42ea-9f50-346e8c7ecb75", "name": "Sticky Note" }, { "parameters": { "content": "## Step 2: Crawl websites using Firecrawl\n ", "height": 480, "width": 1472 }, "type": "n8n-nodes-base.stickyNote", "position": [ -1568, 16 ], "typeVersion": 1, "id": "2a814b9a-7549-4950-89fb-5fa4a5316f10", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 480, "width": 496, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ -96, 16 ], "typeVersion": 1, "id": "5cb975f1-43d3-4a09-8c00-47bf2b64c30a", "name": "Sticky Note2" } ], "connections": { "LLM Crawl Request": { "main": [ [ { "node": "Wait For LLM Crawl", "type": "main", "index": 0 } ] ] }, "Wait For LLM Crawl": { "main": [ [ { "node": "Get LLM Crawl", "type": "main", "index": 0 } ] ] }, "Get LLM Crawl": { "main": [ [ { "node": "LLM TID Field", "type": "main", "index": 0 } ] ] }, "LLM TID Field": { "main": [ [ { "node": "LLM If In Process", "type": "main", "index": 0 } ] ] }, "LLM If In Process": { "main": [ [ { "node": "LLM Result", "type": "main", "index": 0 } ], [ { "node": "Wait For LLM Crawl", "type": "main", "index": 0 } ] ] }, "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "LLM Crawl Request", "type": "main", "index": 0 } ] ] }, "LLM Result": { "main": [ [ { "node": "Code in JavaScript", "type": "main", "index": 0 } ] ] }, "Code in JavaScript": { "main": [ [ { "node": "Crawl Case Study", "type": "main", "index": 0 } ] ] }, "Crawl Case Study": { "main": [ [ { "node": "Wait For LLM Crawl1", "type": "main", "index": 0 } ] ] }, "Wait For LLM Crawl1": { "main": [ [ { "node": "Get LLM Crawl1", "type": "main", "index": 0 } ] ] }, "Get LLM Crawl1": { "main": [ [ { "node": "LLM If In Process1", "type": "main", "index": 0 } ] ] }, "LLM If In Process1": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "Structured Output Parser3": { "ai_outputParser": [ [ { "node": "ICP Scoring", "type": "ai_outputParser", "index": 0 } ] ] }, "OpenAI Chat Model2": { "ai_languageModel": [ [ { "node": "ICP Scoring", "type": "ai_languageModel", "index": 0 } ] ] }, "Case Study Merge": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "Get LLM Crawl2": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "If Score < 3", "type": "main", "index": 0 } ] ] }, "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Get row(s) in sheet", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get LLM Crawl2", "type": "main", "index": 0 } ] ] }, "Qualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Unqualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "If Score < 3": { "main": [ [ { "node": "Unqualified", "type": "main", "index": 0 } ], [ { "node": "Qualified", "type": "main", "index": 0 } ] ] }, "Merge": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "777d6586-3fe0-4245-b004-b280d9cd8841", "activeVersionId": null, "versionCounter": 82, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-03T19:35:42.463Z", "createdAt": "2025-11-03T19:35:42.463Z", "role": "workflow:owner", "workflowId": "VjcOz2TJQTTjtY51", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-20T20:40:04.000Z", "createdAt": "2025-11-07T01:07:25.979Z", "id": "Ea4aMM0cqaWGBb32", "name": "客戶A - Firecrawl", "description": null, "active": false, "isArchived": true, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 496, 112 ], "id": "97f47484-823a-4531-83dc-a131cb1bd83d", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n Firecrawl Input", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 656, 112 ], "id": "008a4c6f-ff8d-4eed-a93d-64bbc9d3c0fd", "name": "Get row(s) in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "batchSize": 5, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 976, 112 ], "id": "b0b98425-b5ee-4aba-a6fe-a9127c7c29a1", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1456, 176 ], "id": "5a7e781e-cba6-4bf5-b8af-4d8756aeb292", "name": "Get Crawl Dataset", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1616, 176 ], "id": "89873866-11ab-4de6-8141-1942d36a453f", "name": "If" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ 816, 112 ], "id": "4548f804-50dc-4850-b13c-1c164ae3008c", "name": "Filter" }, { "parameters": { "content": "## Step 2: Extract data from firecrawl output\n ", "height": 448, "width": 1632 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 640 ], "typeVersion": 1, "id": "aa73bbd8-4f96-4ec0-bfa3-407ad9f44e01", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 448, "width": 464, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ 2000, 640 ], "typeVersion": 1, "id": "bc20f952-fedd-465f-853c-8769397bfdde", "name": "Sticky Note2" }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "const merged = $input.item.json.merged_markdown; // 從前輸出取 merged_markdown\nconst front = merged.slice(0, 1000);\nconst back = merged.slice(-1000);\nconst shortened = front + back;\nreturn { json: { shortened_markdown: shortened } }; // 輸出 shortened_markdown" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 976, 448 ], "id": "8cc6f397-675a-4b63-987f-2298bfcf11fe", "name": "Shorten Markdown", "alwaysOutputData": true }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1936, 192 ], "id": "35350752-5b48-4c90-ade1-97553b96003a", "name": "Get Crawl Dataset1", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2096, 192 ], "id": "50133514-6f89-452f-8f03-231fb1d3d673", "name": "If1" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 448 ], "id": "4485e955-de8e-4850-8c49-bb1125d43805", "name": "Get Crawl Dataset2", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 20 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1776, 192 ], "id": "0dbf76d6-c779-4a2c-b7a7-aff5f4d7ca95", "name": "20 sec", "webhookId": "4daada68-5b51-42b4-8307-3a7a3b160665" }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1296, 176 ], "id": "d5673b94-c43d-4f4e-b74c-72afc56a92dd", "name": "10 sec", "webhookId": "2765ddc6-5b45-496a-8cce-3c3be41ec5e8" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 2032, 448 ], "id": "dcfdfe21-8dd2-4863-8a88-a992b472aafe", "name": "Get Crawl Dataset3", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2192, 448 ], "id": "287a566b-a91a-4564-bc0d-f751178115bb", "name": "If2" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 752 ], "id": "3624e96b-678e-41b7-8182-183bc6663013", "name": "Get Crawl Dataset4", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1872, 448 ], "id": "09f107cc-2ff0-4f39-8b7e-9c337f27d51e", "name": "10 sec1", "webhookId": "2765ddc6-5b45-496a-8cce-3c3be41ec5e8" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ☆ Qualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json[\"First Name\"] }}", "last name": "={{ $('Get row(s) in sheet').item.json[\"Last Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json[\"Person Linkedin Url\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 912 ], "id": "a92ff9be-6cf8-41fc-bcdd-78863ac19664", "name": "Qualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ⨯ Unqualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 736 ], "id": "55222a23-f4b1-4545-ae5e-f67e0481ff45", "name": "Unqualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "cf6d927d-baf8-412b-911a-77d4cfbaf98e", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "1", "operator": { "type": "string", "operation": "equals" } }, { "id": "a71d0bb7-5845-46df-b5d8-d2fab0477bcb", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "2", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "7868fe66-3516-4d01-90e1-32aaafc0f475", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "3", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "24a7144e-0116-4040-b5b1-7b3fc6f46568", "leftValue": "={{ $json.output.ICP }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "ea2c8572-edab-4bbc-9100-29ba53c21b61", "leftValue": "={{ $json.output[\"Service Benefit\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "014459a4-db94-4efa-a4f3-683d8d1470f9", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1840, 752 ], "id": "8b01b93a-a407-4e84-a0c8-e609f84aec4a", "name": "If Score < 3" }, { "parameters": {}, "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 2240, 752 ], "id": "511b8803-bad1-47a4-8f95-1edad9067e29", "name": "Merge" }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $('Filter').item.json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"app/.*\",\n \"demo/.*\",\n \"news/.*\",\n \"events/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"home/.*\",\n \"pricing/.*\",\n \"about/.*\",\n \"products/.*\",\n \"services/.*\",\n \"company/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"benefits/.*\",\n \"integrations/.*\",\n \"target-audience/.*\",\n \"platform/.*\",\n \"why/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1712, 448 ], "id": "2f82ee25-dac5-4d9c-989d-dd95cc718bc6", "name": "Crawl ICP", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"pricing/.*\",\n \"products/.*\",\n \"services/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\",\n \"home/.*\",\n \"about/.*\",\n \"privacy/.*\",\n \"cookies/.*\",\n \"blog-preview/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customers/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1136, 176 ], "id": "e22cf7c6-c6c7-46a9-befd-8ef35f8aa4d9", "name": "Crawl Case Study", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 448 ], "id": "abf3decf-4d35-4e79-8a97-f8a84256e73e", "name": "30 sec", "webhookId": "e3f7a2c6-ad79-4127-b998-d8c3097cdfb1" }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.5-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('Case Study Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for estimated Ideal Customer Profile (ICP) from crawled web page markdown focused on case studies. Your task is to process the provided crawled page markdown and extract an estimated ICP based on client companies mentioned in case studies, outputting it in JSON format inside an \"output\" key.\n\n\"Estimated ICP\": Extract a short phrase describing the ideal customer profile (ICP) as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\"). Base it primarily on client companies found in case studies, inferring their industries or types only if unclear from markdown.\n- Scope: Search ONLY in the provided markdown initially. Prefer sections like: \"Case Studies\", \"Success Stories\", \"Clients\", \"Testimonials\", \"Projects\", \"Portfolio\", \"Our Work\", \"Results\", \"Reviews\".\n- What counts as a case study client: Named companies explicitly presented as clients in case studies, projects, success stories, or testimonials, with descriptions of work done for them. Accept names in headings, titles, attributions, or body text where context indicates client relationship. Always exclude the host company's own name (provided in user input) to avoid self-referencing.\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\").\n - Format: Company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple industries, infer the most common or dominant one based on frequency of case studies, prioritizing precision over breadth.\n - Analyze descriptions: From markdown, identify industries or company types of the clients (e.g., if a case study describes \"helping an e-commerce retailer optimize logistics\", infer \"e-commerce businesses\").\n - If multiple case studies cluster in one industry (e.g., several in finance), set ICP to that (e.g., \"investment banks and asset managers\").\n - Preserve logical grouping; avoid broad terms like \"all businesses\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include the host company's own name, products, partners, or non-client entities. If a company name matches the host company, ignore it entirely.\n - If industry unclear from markdown: As a last resort only (if completely no info after exhaustive analysis), use tools to search the web for the client company's industry (e.g., web_search with query like \"what industry is [company name] in? site:wikipedia.org\" or reliable sources). Limit to 1-2 searches per unclear client, and only for confirmed client names excluding the host company. Integrate results to refine ICP without guessing.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) At least one (preferably multiple) explicit case study clients with context indicating they are customers (e.g., \"We helped [company] achieve X\").\n B) The inferred ICP is based on company types/industries/roles, not products, awards, or vague mentions.\n C) The phrase is contiguous and directly mappable to client descriptions; no guessing.\n D) If searching externally, results must confirm industry without contradiction, and searches must exclude the host company name.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by” – ignore these entirely.\n - Logo walls or lists without case study details or clear client labeling.\n - Ambiguous mentions where companies could be partners/vendors, not clients.\n - Fewer than 2 case studies, or no clear industry clustering.\n - External search yields no reliable industry info or conflicts with markdown.\n- If NOT FOUND, NOT CERTAIN, or no viable ICP: Return an empty string \"\". Do NOT write \"N/A\", \"red\", \"None\", or any explanation. Do not output search suggestions if using tools – integrate directly.\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Estimated ICP\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty.", "codeExecution": false, "maxOutputTokens": 1000, "candidateCount": 1, "temperature": 0.2, "topP": 1, "topK": 40 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1136, 448 ], "id": "4170f344-8831-4e3a-a789-e202438fda02", "name": "Case Study ICP", "alwaysOutputData": false, "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash", "mode": "list", "cachedResultName": "models/gemini-2.5-flash" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $json.merged_markdown }}\nEstimated ICP: {{ $('Case Study ICP').item.json.content.parts[0].text }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.\n\n\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").\n- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow.\n - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth.\n - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit.\n - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include products, partners, or non-customer entities.\n - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort):\n A) Explicit mentions of target customers in relevant sections.\n B) The inferred ICP is based on company types/industries/roles, not vague or broad statements.\n C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence.\n D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\n\n\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").\n- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points).\n - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text.\n - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation.\n - If multiple products/services, select the primary or most emphasized one.\n - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation.\n - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains).\n - Do NOT guess; if unclear, treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) Explicit description of primary product/service and its benefit.\n B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision.\n C) Directly from text; no invention, and it must support logical flow in the target sentence.\n D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).\n\nOverall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.\n\n- Disqualifiers (any one triggers NOT FOUND for the field):\n - Vague or no explicit mentions in preferred sections.\n - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages.\n - Ambiguous or conflicting info.\n- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.\n\nFinal Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 3000, "candidateCount": 1 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 976, 752 ], "id": "94ff13f7-1c0a-4ee7-921a-3c6d3d6d4542", "name": "ICP + Service", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('ICP Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are an ICP scoring system for identifying B2B SaaS companies that align with high-ticket deals ($25k+). Analyze the provided crawled markdown from the company's website to determine fit based on their services, target audience, and any visible pricing indicators. Objective: Identify companies offering B2B SaaS solutions targeted at businesses, excluding B2C-focused SaaS, advertising/marketing/sales software, e-commerce software or marketplaces, managed IT services/staffing, higher education SaaS, events/planners/conferences SaaS, Web3/blockchain/cryptocurrency software, government SaaS, Salesforce partners, medical billing services/software, or clinic/medical scheduling software. Prioritize indications of higher-ticket pricing (avoid low monthly plans like $250-500/month if mentioned; favor custom/enterprise pricing).\n\nResearch Method:\n\n1. Review the crawled markdown content from the company's website, focusing on sections like homepage, about us, products/services, solutions, case studies, testimonials, pricing, and blog.\n\n2. Verify service alignment: Scan for keywords indicating B2B SaaS (e.g., \"enterprise software\", \"business solutions\", \"B2B platform\"). Confirm services are for businesses, not the avoided categories. For example:\n - Positive: Mentions \"B2B tools for companies\", \"enterprise automation\".\n - Negative: Keywords like \"B2C app\", \"marketing software\", \"sales CRM\", \"ecommerce platform\", \"IT managed services\", \"staffing solutions\", \"edtech\", \"event management\", \"blockchain software\", \"govtech\", \"Salesforce partner\", \"medical billing\", \"clinic scheduling\".\n\n3. Infer target audience: Analyze case studies, client testimonials, or \"who we serve\" sections for business/enterprise focus. For example:\n - Positive: Clients as \"businesses in tech/finance\", \"enterprise users\".\n - Negative: Mentions \"consumers\", \"e-commerce stores\", \"universities\", \"event planners\", \"government agencies\", \"healthcare clinics\".\n\n4. Check for pricing clues: Look for mentions like \"starting at $X/month\" or \"enterprise pricing\". For example:\n - Positive: \"Custom pricing\", \"contact for quote\", \"annual contracts for enterprises\" (suggesting $25k+).\n - Negative: \"Starting at $250-500/month\", \"low-cost plans\" (indicating low-ticket).\n\n5. Overall fit assessment: Weigh positive vs. negative indicators. If ambiguous, lean toward moderate scores rather than low.\n\nClassification: Rate on a scale from 1 to 10:\n- 1–3: Not a fit (use only if unavoidable or definitively matches an avoided category, e.g., clearly in avoid list like B2C SaaS or low-ticket with explicit $250-500/month pricing; reserve for confirmed mismatches).\n- 4–6: Moderate fit (e.g., some B2B elements but potential overlap with avoided areas, unclear audience, or mixed pricing signals).\n- 7–8: Good fit (e.g., strong B2B SaaS focus, no avoids, suggests higher pricing).\n- 9–10: Perfect fit (e.g., high-ticket B2B SaaS with clear business targeting, no negatives).\n\nDeliverable: Provide ONLY the rating for the company on a scale from 1 to 10, with no additional explanation or details.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty. If no viable score (e.g., insufficient data), return \"0\".", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1264, 752 ], "id": "cb30c570-5030-4f6f-94e1-9579e0049a16", "name": "ICP Scoring", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1424, 448 ], "id": "3c802f09-80b1-42a2-9cf4-8cc2043b6a47", "name": "Format Names", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 448 ], "id": "c03ce62f-b06a-4434-987d-413d8772fa5f", "name": "Case Study Merge", "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 752 ], "id": "de42c4a6-00d1-444e-9c36-b95aa3146e00", "name": "ICP Merge", "alwaysOutputData": true }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 752 ], "id": "b9768eb3-e059-4632-b66b-90b57e809a7e", "name": "30 sec1", "webhookId": "4daada68-5b51-42b4-8307-3a7a3b160665" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "responseFormat": "json_object", "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1552, 912 ], "id": "4b4b75d9-d096-4190-a724-09ca6633264f", "name": "OpenAI Chat Model", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "jsonSchemaExample": "{\n\t\"ICP\": \"iterative hardware teams\",\n \"Service Benefit\": \"accelerate talent sourcing\",\n \"Company Name\": \"Cyberleaf\",\n \"Job Title\": \"Regional Director Sales\",\n \"ICP Score\": \"8\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1696, 912 ], "id": "844de2de-d1b8-4e81-aa9e-bb2d40868dc9", "name": "Structured Output Parser" }, { "parameters": { "promptType": "define", "text": "=Process these input strings and extract the fields ICP, Service Benefit, Company Name, Job Title, ICP Score into a clean single JSON object:\n{{ $json.content.parts[0].text }}\n{{ $('ICP + Service').item.json.content.parts[0].text}}\n{{ $('Format Names').item.json.content.parts[0].text }}", "hasOutputParser": true, "options": { "systemMessage": "You are a JSON data cleaner and organizer for extracting and structuring specific fields from input strings. \n\nYour task is to process the provided input strings, which may contain JSON objects or arrays, and extract values for the fields: ICP, Service Benefit, Company Name, Job Title, ICP Score. \n\nObjective: \nTransform the inputs into a clean single object that merges all extracted fields from the inputs, filling missing fields with empty strings. \n\nResearch Method: \n1. Parse each input string as JSON, handling potential formatting issues like escaped newlines. \n2. For each parsed structure, navigate to the innermost object (e.g., under \"output\") and extract the specified fields if present. \n3. If an input is an array, process each element separately. \n4. Merge all extracted data into a single object, overwriting duplicates if any (use the last value encountered). \n5. If parsing fails or no fields match, leave the corresponding field as an empty string in the output object. \n\nClassification: \nEnsure fields are extracted exactly as strings; do not infer or add data. \n\nDeliverable: \nProvide ONLY the extracted data as a single object, with no additional explanation or details. \n\nOutput format: \nReturn ONLY a JSON object in this exact structure: {\"ICP\": \"value or empty string\", \"Service Benefit\": \"value or empty string\", \"Company Name\": \"value or empty string\", \"Job Title\": \"value or empty string\", \"ICP Score\": \"value or empty string\"}. No additional text, commentary, or formatting. Always output a single object, even if empty or with empty strings." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1552, 752 ], "id": "3ba69f6d-2906-4c7b-9306-a0494f659c3c", "name": "Clean" }, { "parameters": { "content": "## Step 1: Crawl websites using firecrawl\n", "height": 608, "width": 2096, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 32 ], "typeVersion": 1, "id": "fca4c2ff-5f13-4e2b-af1b-ece4710e09a9", "name": "Sticky Note" } ], "connections": { "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Get row(s) in sheet", "type": "main", "index": 0 } ] ] }, "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "Crawl Case Study", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset": { "main": [ [ { "node": "If", "type": "main", "index": 0 } ] ] }, "If": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "20 sec", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Shorten Markdown": { "main": [ [ { "node": "Case Study ICP", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset1": { "main": [ [ { "node": "If1", "type": "main", "index": 0 } ] ] }, "If1": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset2": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ] ] }, "20 sec": { "main": [ [ { "node": "Get Crawl Dataset1", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get Crawl Dataset", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset3": { "main": [ [ { "node": "If2", "type": "main", "index": 0 } ] ] }, "If2": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec1", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset4": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ] ] }, "10 sec1": { "main": [ [ { "node": "Get Crawl Dataset3", "type": "main", "index": 0 } ] ] }, "Qualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Unqualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "If Score < 3": { "main": [ [ { "node": "Unqualified", "type": "main", "index": 0 } ], [ { "node": "Qualified", "type": "main", "index": 0 } ] ] }, "Merge": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Crawl ICP": { "main": [ [ { "node": "10 sec1", "type": "main", "index": 0 } ] ] }, "Crawl Case Study": { "main": [ [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "30 sec": { "main": [ [ { "node": "Get Crawl Dataset2", "type": "main", "index": 0 } ] ] }, "Case Study ICP": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] }, "ICP + Service": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "Clean", "type": "main", "index": 0 } ] ] }, "Format Names": { "main": [ [ { "node": "Crawl ICP", "type": "main", "index": 0 } ] ] }, "Case Study Merge": { "main": [ [ { "node": "Shorten Markdown", "type": "main", "index": 0 } ] ] }, "ICP Merge": { "main": [ [ { "node": "ICP + Service", "type": "main", "index": 0 } ] ] }, "30 sec1": { "main": [ [ { "node": "Get Crawl Dataset4", "type": "main", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Clean", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "Clean", "type": "ai_outputParser", "index": 0 } ] ] }, "Clean": { "main": [ [ { "node": "If Score < 3", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "b558afdd-1d60-45b4-b0a2-daae80d23e6e", "activeVersionId": null, "versionCounter": 70, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-07T01:07:25.983Z", "createdAt": "2025-11-07T01:07:25.983Z", "role": "workflow:owner", "workflowId": "Ea4aMM0cqaWGBb32", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-22T19:50:15.000Z", "createdAt": "2025-11-08T05:55:57.949Z", "id": "1wTlVvSGibxy80WY", "name": "Test", "description": null, "active": false, "isArchived": false, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 0, 0 ], "id": "38ec430f-0bcc-4c39-83d4-2368f2a20fba", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "jsonSchemaExample": "{\n\t\"ICP\": \"e-commerce businesses\",\n \t\"Service Benefit\": \"streamline delivery services\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 976, 160 ], "id": "50dd671a-f3ea-4d7f-a3af-867d3fc00f8c", "name": "Structured Output Parser" }, { "parameters": { "jsonSchemaExample": "{\n\t\"Estimated ICP\": \"e-commerce businesses\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 688, 160 ], "id": "da90078c-afa7-48cc-a144-710f5cee7fe0", "name": "Structured Output Parser2" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $('Case Study').item.json[\"Case Study\"][0].merged_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are a precise data extractor for estimated Ideal Customer Profile (ICP) from crawled web page markdown focused on case studies. Your task is to process the provided crawled page markdown and extract an estimated ICP based on client companies mentioned in case studies, outputting it in JSON format inside an \"output\" key.\n\n\"Estimated ICP\": Extract a short phrase describing the ideal customer profile (ICP) as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\"). Base it primarily on client companies found in case studies, inferring their industries or types only if unclear from markdown.\n- Scope: Search ONLY in the provided markdown initially. Prefer sections like: \"Case Studies\", \"Success Stories\", \"Clients\", \"Testimonials\", \"Projects\", \"Portfolio\", \"Our Work\", \"Results\", \"Reviews\".\n- What counts as a case study client: Named companies explicitly presented as clients in case studies, projects, success stories, or testimonials, with descriptions of work done for them. Accept names in headings, titles, attributions, or body text where context indicates client relationship. Always exclude the host company's own name (provided in user input) to avoid self-referencing.\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\").\n - Format: Company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple industries, infer the most common or dominant one based on frequency of case studies, prioritizing precision over breadth.\n - Analyze descriptions: From markdown, identify industries or company types of the clients (e.g., if a case study describes \"helping an e-commerce retailer optimize logistics\", infer \"e-commerce businesses\").\n - If multiple case studies cluster in one industry (e.g., several in finance), set ICP to that (e.g., \"investment banks and asset managers\").\n - Preserve logical grouping; avoid broad terms like \"all businesses\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include the host company's own name, products, partners, or non-client entities. If a company name matches the host company, ignore it entirely.\n - If industry unclear from markdown: As a last resort only (if completely no info after exhaustive analysis), use tools to search the web for the client company's industry (e.g., web_search with query like \"what industry is [company name] in? site:wikipedia.org\" or reliable sources). Limit to 1-2 searches per unclear client, and only for confirmed client names excluding the host company. Integrate results to refine ICP without guessing.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) At least one (preferably multiple) explicit case study clients with context indicating they are customers (e.g., \"We helped [company] achieve X\").\n B) The inferred ICP is based on company types/industries/roles, not products, awards, or vague mentions.\n C) The phrase is contiguous and directly mappable to client descriptions; no guessing.\n D) If searching externally, results must confirm industry without contradiction, and searches must exclude the host company name.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by” – ignore these entirely.\n - Logo walls or lists without case study details or clear client labeling.\n - Ambiguous mentions where companies could be partners/vendors, not clients.\n - Fewer than 2 case studies, or no clear industry clustering.\n - External search yields no reliable industry info or conflicts with markdown.\n- If NOT FOUND, NOT CERTAIN, or no viable ICP: Return an empty string \"\". Do NOT write \"N/A\", \"red\", \"None\", or any explanation. Do not output search suggestions if using tools – integrate directly.\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Estimated ICP\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 544, 0 ], "id": "4aef86c7-5928-4b9d-a0ec-858e2f7d7a17", "name": "Case Study ICP", "alwaysOutputData": false }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $('ICP').item.json.ICP[0].merged_markdown }}\nEstimated ICP: {{ $json.output['Estimated ICP'] }}", "hasOutputParser": true, "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.\n\n\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").\n- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow.\n - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth.\n - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit.\n - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include products, partners, or non-customer entities.\n - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort):\n A) Explicit mentions of target customers in relevant sections.\n B) The inferred ICP is based on company types/industries/roles, not vague or broad statements.\n C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence.\n D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\n\n\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").\n- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points).\n - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text.\n - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation.\n - If multiple products/services, select the primary or most emphasized one.\n - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation.\n - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains).\n - Do NOT guess; if unclear, treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) Explicit description of primary product/service and its benefit.\n B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision.\n C) Directly from text; no invention, and it must support logical flow in the target sentence.\n D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).\n\nOverall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.\n\n- Disqualifiers (any one triggers NOT FOUND for the field):\n - Vague or no explicit mentions in preferred sections.\n - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages.\n - Ambiguous or conflicting info.\n- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.\n\nFinal Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 832, 0 ], "id": "9f638dab-d3ba-4937-ba8a-53d8508f81c2", "name": "ICP + Service", "alwaysOutputData": false }, { "parameters": { "jsonSchemaExample": "{\n \"ICP Score\": \"7\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1264, 160 ], "id": "a96db530-ee4c-4a4d-8fe5-916a72ec3965", "name": "Structured Output Parser3" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $('ICP').item.json.ICP[0].merged_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are an ICP scoring system for identifying B2B SaaS companies that align with high-ticket deals ($25k+). Analyze the provided crawled markdown from the company's website to determine fit based on their services, target audience, and any visible pricing indicators. Objective: Identify companies offering B2B SaaS solutions targeted at businesses, excluding B2C-focused SaaS, advertising/marketing/sales software, e-commerce software or marketplaces, managed IT services/staffing, higher education SaaS, events/planners/conferences SaaS, Web3/blockchain/cryptocurrency software, government SaaS, Salesforce partners, medical billing services/software, or clinic/medical scheduling software. Prioritize indications of higher-ticket pricing (avoid low monthly plans like $250-500/month if mentioned; favor custom/enterprise pricing).\n\nResearch Method:\n\n1. Review the crawled markdown content from the company's website, focusing on sections like homepage, about us, products/services, solutions, case studies, testimonials, pricing, and blog.\n\n2. Verify service alignment: Scan for keywords indicating B2B SaaS (e.g., \"enterprise software\", \"business solutions\", \"B2B platform\"). Confirm services are for businesses, not the avoided categories. For example:\n - Positive: Mentions \"B2B tools for companies\", \"enterprise automation\".\n - Negative: Keywords like \"B2C app\", \"marketing software\", \"sales CRM\", \"ecommerce platform\", \"IT managed services\", \"staffing solutions\", \"edtech\", \"event management\", \"blockchain software\", \"govtech\", \"Salesforce partner\", \"medical billing\", \"clinic scheduling\".\n\n3. Infer target audience: Analyze case studies, client testimonials, or \"who we serve\" sections for business/enterprise focus. For example:\n - Positive: Clients as \"businesses in tech/finance\", \"enterprise users\".\n - Negative: Mentions \"consumers\", \"e-commerce stores\", \"universities\", \"event planners\", \"government agencies\", \"healthcare clinics\".\n\n4. Check for pricing clues: Look for mentions like \"starting at $X/month\" or \"enterprise pricing\". For example:\n - Positive: \"Custom pricing\", \"contact for quote\", \"annual contracts for enterprises\" (suggesting $25k+).\n - Negative: \"Starting at $250-500/month\", \"low-cost plans\" (indicating low-ticket).\n\n5. Overall fit assessment: Weigh positive vs. negative indicators. If ambiguous, lean toward moderate scores rather than low.\n\nClassification: Rate on a scale from 1 to 10:\n- 1–3: Not a fit (use only if unavoidable or definitively matches an avoided category, e.g., clearly in avoid list like B2C SaaS or low-ticket with explicit $250-500/month pricing; reserve for confirmed mismatches).\n- 4–6: Moderate fit (e.g., some B2B elements but potential overlap with avoided areas, unclear audience, or mixed pricing signals).\n- 7–8: Good fit (e.g., strong B2B SaaS focus, no avoids, suggests higher pricing).\n- 9–10: Perfect fit (e.g., high-ticket B2B SaaS with clear business targeting, no negatives).\n\nDeliverable: Provide ONLY the rating for the company on a scale from 1 to 10, with no additional explanation or details.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty. If no viable score (e.g., insufficient data), return empty string \"\"." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1120, 0 ], "id": "842b221c-a706-4f83-8e67-75274b1ba2cb", "name": "ICP Scoring" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-mini", "mode": "list", "cachedResultName": "gpt-5-mini" }, "options": { "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 544, 160 ], "id": "bea477b7-4eaa-4273-92f5-f01a3b18118c", "name": "OpenAI Chat Model", "credentials": { "openAiApi": { "id": "UggzGrwKVc28ZAlJ", "name": "Local" } } }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-mini", "mode": "list", "cachedResultName": "gpt-5-mini" }, "options": { "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 832, 160 ], "id": "777e1621-ee34-466a-994f-fe2f09dcc0f9", "name": "OpenAI Chat Model1", "credentials": { "openAiApi": { "id": "UggzGrwKVc28ZAlJ", "name": "Local" } } }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-mini", "mode": "list", "cachedResultName": "gpt-5-mini" }, "options": { "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1120, 160 ], "id": "aeadf2ef-42b5-4d33-a92e-ee713ded888b", "name": "OpenAI Chat Model2", "credentials": { "openAiApi": { "id": "UggzGrwKVc28ZAlJ", "name": "Local" } } }, { "parameters": { "assignments": { "assignments": [ { "id": "e16ac8fd-fc1b-4064-9817-70e44457b868", "name": "Case Study", "value": "[ { \"merged_markdown\": \"Flow raises **$23M Series A** from ![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg)\\n\\n[Read More](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n* [Handbook](https://flowengineering.com/handbook/volume-1/)\\n \\n* [AI](https://flowengineering.com/ai-agent/)\\n \\n* Industries\\n \\n * [Launch Vehicles](https://flowengineering.com/launch-vehicles/)\\n \\n * [Satellites](https://flowengineering.com/satellites/)\\n \\n * [Ground Station](https://flowengineering.com/ground-station/)\\n \\n * [Aviation & Aero](https://flowengineering.com/aviation-and-aero/)\\n \\n * [Fusion](https://flowengineering.com/fusion/)\\n \\n * [Nuclear (SMR)](https://flowengineering.com/nuclear/)\\n \\n * [Quantum](https://flowengineering.com/quantum/)\\n \\n * [eVTOL](https://flowengineering.com/evtol/)\\n \\n * [Automotive](https://flowengineering.com/automotive/)\\n \\n * [Robotics](https://flowengineering.com/robotics/)\\n \\n * [Medical Devices](https://flowengineering.com/medical-devices/)\\n \\n* Product\\n \\n * [Requirements Management](https://flowengineering.com/requirements-management/)\\n \\n * [Architecture](https://flowengineering.com/architecture/)\\n \\n * [Traceability](https://flowengineering.com/traceability/)\\n \\n * [Continuous Verification](https://flowengineering.com/continuous-verification/)\\n \\n * [Live V&V](https://flowengineering.com/live-vv/)\\n \\n * [Test Cases\\\\\\n \\\\\\n new](https://flowengineering.com/test-cases/)\\n \\n * [Regulatory](https://flowengineering.com/regulatory/)\\n \\n * [Integrations](https://flowengineering.com/integrations/)\\n \\n* [Pricing](https://flowengineering.com/pricing/)\\n \\n* Resources\\n \\n * [Blog](https://flowengineering.com/blog/)\\n \\n * [Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n \\n * [Security](https://flowengineering.com/security/)\\n \\n* Company\\n \\n * [About us](https://flowengineering.com/about-us/)\\n \\n * [Careers](https://flowengineering.com/careers/)\\n \\n\\nRequirements tool built for rapid development\\n=============================================\\n\\nThe first tool custom built for iterative, cross-functional engineering teams to design & iterate faster\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\n[![Mach Industries](https://images.ctfassets.net/c2mtbunjxyfe/50nWLJRL9wtu5NVk8EqFDc/e351c326e2410330a78c91c63081d583/5b615526-0071-4c7b-96af-11672b4a124a.svg?w=256)](https://machindustries.co/)\\n[![Radiant](https://images.ctfassets.net/c2mtbunjxyfe/6OiITxBbNx2mEabwnUAtlK/5d0a47c9527eb718432fc3c133c0903c/Radiant.png?w=1920)](https://www.radiantnuclear.com/)\\n[![Joby](https://images.ctfassets.net/c2mtbunjxyfe/4cvUNWFbixz8L6FQGCfuck/5ff7b7a2684d84e40162b828bd453d72/Joby.svg?w=1920)](https://www.jobyaviation.com/)\\n[![Rivian](https://images.ctfassets.net/c2mtbunjxyfe/VOLksYa01iqJIBnYtTwar/86203dce1085fa6ae8a3b3f16039bc2b/rivian-logo.svg?w=1080)](https://rivian.com/)\\n[![Astranis](https://images.ctfassets.net/c2mtbunjxyfe/5ZZHV79zARsLZkorcUaA2H/a1dcb0f27bc8dc155e60fd95417e1a0d/Astranis.svg?w=1920)](https://www.astranis.com/)\\n[![Impulse Space](https://images.ctfassets.net/c2mtbunjxyfe/71rebwlPyl8Z54GMToFGuT/7a68d1e327dfc16e8ee102515a0394ae/impulse-space.svg?w=1080)](https://www.impulsespace.com/)\\n[![Stoke](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)](https://www.stokespace.com/)\\n\\n![background-flow-requirements](https://images.ctfassets.net/c2mtbunjxyfe/4LI2XPWn5Vpst3k4hE90v/7061a049ea2c51554243bdd4e750bd02/Group_201.jpg?w=3840)![background-flow-systems](https://images.ctfassets.net/c2mtbunjxyfe/18L2uQ5YXQ0jSN8Jj9N205/e40ad32fe1d79b3c7a79052b5b0e142e/Group_203.jpg?w=3840)![background-flow-architecture](https://images.ctfassets.net/c2mtbunjxyfe/6Ngm6MDAw5sPBZSpwv81x8/bdb70e16d9850b21e155a15dc6cc8f10/Group_200.jpg?w=3840)![background-flow-live-v-and-v](https://images.ctfassets.net/c2mtbunjxyfe/1apz8Vk8WhQptrrI6UnBP5/8e4bd187830042229ff0860ba6f16c28/Group_202.jpg?w=3840)![background-test-cases](https://images.ctfassets.net/c2mtbunjxyfe/4cdwdvMPjh8MSo7XT97SLM/de4ca9c49fc9a209dad9cc9409ce7aa5/background-test-cases.jpg?w=3840)![background-flow-requirements](https://images.ctfassets.net/c2mtbunjxyfe/49DeRgSdpfyUKG7b6gyWmA/731fad76a91d4aabf133fe8ad344d3e1/Group_201.jpg?w=3840)\\n\\nEnd-to-end traceability of requirements & design parameters from design to release\\n\\nEasy for non-systems engineers to collaborate seamlessly\\n\\nA unified view of your systems and interfaces that everyone can use.\\n\\nSee live status of requirements with integrations to design and test data. CI/CD for systems engineering.\\n\\nStreamline test orchestration with a clear, visual representation of all test runs and statuses.\\n\\nFlow is an AI native systems platform. Our agent works on top of your most important design & test data so that you speed up critical work\\n\\n![placeholder Requirements](https://images.ctfassets.net/c2mtbunjxyfe/5Jk2nVPSmbyKzhNnViTBQt/abcfbdc47b11eec402c8535aca96fc2d/ezgif-frame-001.jpg?w=3840)\\n\\n![placeholder Systems](https://images.ctfassets.net/c2mtbunjxyfe/3ltfqP2armZBLwuCyHG2Ci/0ac69a7d215781817bb5441b89f85756/ezgif-frame-001__2_.jpg?w=3840)\\n\\n![placeholder Architecture](https://images.ctfassets.net/c2mtbunjxyfe/109KeEQAybmHEuXsWmLeKO/c72275973bd3a52793136f0b77dbd607/ezgif-frame-001__3_.jpg?w=3840)\\n\\n![placeholder Integrations](https://images.ctfassets.net/c2mtbunjxyfe/1xwp3k67x2msEXStKYMddE/b4d2fdeef6cba0aeb2c05580b02276a7/ezgif-frame-001__4_.jpg?w=3840)\\n\\n![placeholder Test Cases](https://images.ctfassets.net/c2mtbunjxyfe/35UwA6i6nbtWYWpKFn7NZW/b2c791f525302300282493c2ba6ed20a/Screenshot_2025-01-09_at_17.59.29.png?w=3840)\\n\\n![placeholder AI Agent](https://images.ctfassets.net/c2mtbunjxyfe/3eaPVEwdd7BS2TjmGjkbaT/b4b9eb1ce4d83f03717001dd1f1c4030/ezgif-frame-001__1_.jpg?w=3840)\\n\\nBuilt for Iterative Hardware teams\\n\\n[![Rivian](https://images.ctfassets.net/c2mtbunjxyfe/VOLksYa01iqJIBnYtTwar/86203dce1085fa6ae8a3b3f16039bc2b/rivian-logo.svg?w=1080)](https://rivian.com/)\\n[![Joby](https://images.ctfassets.net/c2mtbunjxyfe/2drKCxYRGI6EYXLmRhmtNc/ca8a72e01bb837728a4eb2cbfdf8e50c/Joby.svg?w=1920)](https://www.jobyaviation.com/)\\n[![Astranis](https://images.ctfassets.net/c2mtbunjxyfe/1lG6B3ebgpsYO84jOFJxLt/6d38b98e8991ce21cbf012c1596e4880/Astranis.svg?w=1920)](https://www.astranis.com/)\\n[![Impulse Space](https://images.ctfassets.net/c2mtbunjxyfe/71rebwlPyl8Z54GMToFGuT/7a68d1e327dfc16e8ee102515a0394ae/impulse-space.svg?w=1080)](https://www.impulsespace.com/)\\n[![Stoke](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)](https://www.stokespace.com/)\\n[![Radiant Nuclear](https://images.ctfassets.net/c2mtbunjxyfe/28srfH0gVZQWpTDFjyQ4Xe/055492ff60c71fff494b12a6129982be/Group_661.png?w=1920)](https://www.radiantnuclear.com/)\\n[![Skydio](https://images.ctfassets.net/c2mtbunjxyfe/1diwTaRqyYNrpjgr6jENLK/410f01fc04ab515538ad87a93fef638e/skydio.svg?w=1080)](https://www.skydio.com/)\\n[![Mach Industries](https://images.ctfassets.net/c2mtbunjxyfe/50nWLJRL9wtu5NVk8EqFDc/e351c326e2410330a78c91c63081d583/5b615526-0071-4c7b-96af-11672b4a124a.svg?w=256)](https://machindustries.co/)\\n[![Firehawk Aerospace](https://images.ctfassets.net/c2mtbunjxyfe/SqdXXZevmgohRAwexIPKX/bce0d7192b9eab97acea35eff7516696/firehawk.svg?w=1080)](https://firehawkaerospace.com/)\\n[![Antares Industries](https://images.ctfassets.net/c2mtbunjxyfe/1gzMZ0lvyjnwOLo9aLNO1g/94271e6f85a5748159645906203cca24/Group_659.png?w=1200)](https://antaresindustries.com/)\\n[![Aalyria](https://images.ctfassets.net/c2mtbunjxyfe/4RkdCzDNqGjfqDtf7AHVnf/56b0bc19066fe06cfc0c9ca02f683dc0/Aalyria.svg?w=1920)](https://www.aalyria.com/)\\n[![Harbinger Motor](https://images.ctfassets.net/c2mtbunjxyfe/7hl1ezrBNAfQlNLNe7r413/345d53ce65370c037bb1395eada99bd7/harbinger-logo_02_2.svg?w=1200)](https://harbingermotors.com/)\\n[![Paebbl](https://images.ctfassets.net/c2mtbunjxyfe/IPcWhM1mDxCNeIQ0eLOxJ/91bb611b159c30a8b86b5aaacbd2d334/Paebbl_Lockup_CB_1.svg?w=1080)](https://paebbl.com/)\\n[![Hyphen](https://images.ctfassets.net/c2mtbunjxyfe/3Pc2fRHjFBduWqgJSQ4iTz/6352754e588dd28a10cec60cabc547a5/Hyphen_Logo.svg?w=3840)](https://usehyphen.com/)\\n[![Gravitics](https://images.ctfassets.net/c2mtbunjxyfe/1MBeMlKw9cNX5uIpVTl9py/f5d2d8ec439d2ac5aef97849ba547bb6/gravitics.svg?w=1920)](https://www.gravitics.com/)\\n[![Space Machines Company](https://images.ctfassets.net/c2mtbunjxyfe/1vRV1ZuPkHw17jvS2mkrhT/e8a503d56f94606d692e9d9230df1108/spacemachines.svg?w=1080)](https://www.spacemachines.co/)\\n[![Turion Space](https://images.ctfassets.net/c2mtbunjxyfe/644448yAD8DDlYB1CGXmAH/9cf2937db8adda33387faa34bd150622/Group_662.png?w=1920)](https://turionspace.com/#contact)\\n[![Openstar](https://images.ctfassets.net/c2mtbunjxyfe/1y4lEFfq5rKoKfQI7GsBJD/06a519aa84916ce60673fd74ef3d6e9d/Openstar.svg?w=1920)](https://www.openstar.tech/)\\n[![Phase Four](https://images.ctfassets.net/c2mtbunjxyfe/5OVBKR4fwGwRFvMUSzDjgs/6cd1252895bf4a725a21ef84ecbc5bc3/phasefour.svg?w=1080)](https://phasefour.io/)\\n[![Intramotev](https://images.ctfassets.net/c2mtbunjxyfe/38IKMTgonUFuKTcdAY8bNA/2a0d2ce31dba8850f25aeb196aca8147/Frame_15.png?w=1920)](https://intramotev.com/)\\n\\n**Iterate faster**. Integrate your design tools with requirements for live V&V tracking.\\n========================================================================================\\n\\nCollaborateParameterizeIntegrateVerifyRelease\\n\\n#### **Collaborate**. Give engineers real ownership of requirements. Help everyone take a system first mindset.\\n\\n![collaborative](https://images.ctfassets.net/c2mtbunjxyfe/6f30mkFgsXMwsTqPEbfEbN/24794e454d49c56031490bf1bc1e892a/collaborative.jpg?w=3840)\\n\\n#### **Parameterize**. Convert text-based requirements into machine readable checks for budgets or actuals.\\n\\n![parametrise](https://images.ctfassets.net/c2mtbunjxyfe/7yASuu0i2X3Pb1uULZypux/1f6cb3f6444b333223f86796849aa7c7/parametrise.jpg?w=3840)\\n\\n#### **Integrate**. Drive parameters via API's or integrations for live V&V tracking.\\n\\n![integrate](https://images.ctfassets.net/c2mtbunjxyfe/1K50IkC6c3Prc2gHw2Kfkw/e4076b0e6ca32498df5b9b8d96d68d77/integrate.jpg?w=3840)\\n\\n#### **Verify**. Continuously verify your design on every major design change saving weeks of manual checks.\\n\\n![verify](https://images.ctfassets.net/c2mtbunjxyfe/3JZTjJQhRsUtJqNMvVYLPp/c23009a771f80bcf50bc8fdae94c9ca7/verify.jpg?w=3840)\\n\\n#### **Release**. Generate reports for regulators with end to end design traceability.\\n\\n![release](https://images.ctfassets.net/c2mtbunjxyfe/3bVRNAWQAHVtqIxr6MTMRD/d21e8c4c231b58edd3437335699db157/release.jpg?w=3840)\\n\\nIntegrate, propagate and manage change in minutes\\n=================================================\\n\\nWithout FlowWith Flow\\n\\nNo clean way to communicate requirements\\n\\nEveryone is out of sync\\n\\nFragile, unscalable and error prone spreadsheets\\n\\nEngineering team working in isolation\\n\\nDifficult to manage and see interdependencies\\n\\n![without flow](https://images.ctfassets.net/c2mtbunjxyfe/4t9nq21if0SU0uv0TAokp1/699af5cdefdae9372d6aba725e2a0674/without-flow-export.png?w=3840)\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)\\n\\nFlow is the requirements management tool of our dreams\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nFlow is the requirements management tool of our dreams. It reflects how we work—fast, iterative, and focused on giving everyone a common, integrated view of our system.\\n\\nGetting to CI/CD for systems is the dream. Flow is what finally got us there. It’s not just another tool, it’s the backbone of how we build now.\\n\\n**We’ve had new hires tell us, unprompted, ‘this is the best requirements tool I’ve ever used’.** That says it all.\\n\\n![Ryan McCullough](https://images.ctfassets.net/c2mtbunjxyfe/BwghBr8T2Rac5c8Bp4lll/954ef86b58d1559d131a40ce62517fa2/1674681902140.jpeg?w=96)\\n\\nRyan McCullough\\n\\nSystem Architect\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/4RkdCzDNqGjfqDtf7AHVnf/56b0bc19066fe06cfc0c9ca02f683dc0/Aalyria.svg?w=1920)\\n\\nThere’s nothing on the market that compares\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nWe needed a tool that could keep up with the pace of our organization, adapt to the increasing complexity of our products, and remain easy to manage without full time dedicated resources. Flow has delivered.\\n\\nUnlike many tools that disrupt workflows or create bottlenecks, **Flow integrates successfully with how modern engineering teams operate.** It’s fast, flexible, and scales with ease.\\n\\nIn my experience, there’s nothing on the market that compares.\\n\\n![Joe Barton](https://images.ctfassets.net/c2mtbunjxyfe/fWufVVTUhdrZ9s933sdlr/ddbafed64eed1d1624b8910f1591af7f/1563737889020.jpeg?w=96)\\n\\nJoe Barton\\n\\nTechnical Program Manager\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/1vRV1ZuPkHw17jvS2mkrhT/e8a503d56f94606d692e9d9230df1108/spacemachines.svg?w=1080)\\n\\nI have not found anything comparable to Flow in giving us flexibility without compromising functionality\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nBefore Flow, this meant juggling spreadsheets, duplicate docs, and months of rework. Now it just works. I have not found anything comparable to Flow in giving us **flexibility without compromising functionality**.\\n\\nWith Flow, all our requirements, design targets, and simulation data actually live in one place. That might not sound revolutionary—but if you’ve ever tried to do this in hardware, you know how painful it actually is.\\n\\n![Rajat](https://images.ctfassets.net/c2mtbunjxyfe/6OyqfjAtOwDESMXXfyFUVT/ba34a3c74a936e63bd1314fab803cb76/1516229306173.jpeg?w=96)\\n\\nRajat Kulshrestha\\n\\nCEO & Co-Founder, Space Machines Company\\n\\n1 / 3\\n\\nBuilt for cross-functional teams\\n\\nWhere iterative teams meet regulatory compliance\\n================================================\\n\\nSystem Engineers\\n\\nChief Engineers\\n\\nResponsible Engineers\\n\\nSystem Engineers | Levels\\n=========================\\n\\nSpend less time chasing down requirements updates from your engineers and spend more time optimizing your system to build better products.\\n\\n* Save dozens of hours for each design review\\n* Automated verification from your team & tools\\n* Identify problems before it's too late\\n\\n![System Engineers | Levels](https://images.ctfassets.net/c2mtbunjxyfe/6B4WqtlLlkzknpLJYOXFo8/653b810b3df2a83afb5078bf24a0c3b0/Property_1_Default.png?w=3840)\\n\\nFlexible but powerful\\n=====================\\n\\nReplace the hassle of manually updating spreadsheets or using cumbersome enterprise tools by upgrading your workflow with collaborative requirements management that's always up to date, for everyone.\\n\\nAgile and scalable\\n\\n![Flow](https://images.ctfassets.net/c2mtbunjxyfe/2Zqj08RbDH0XRgIfZYdky3/a936fdee09982e7d1c537b58cb07411c/Vector.png?w=48)Flow\\n\\nExpensive legacy tools\\n\\n![Enterprise](https://images.ctfassets.net/c2mtbunjxyfe/4Ao6Bzcbn3Q1qYe7bbARwa/a6232f8f08f2ea67c25d0b9fcac7d105/ibm-rational-doors_215495.webp?w=48)IBM Doors\\n\\nITAR Compliance\\n\\nContinuous Verification\\n\\nBaselines, config, history\\n\\nAccessible & Collaborative\\n\\nMBSE & Architecture\\n\\nFast, Flexible & Easy\\n\\nCustomization\\n\\nOnboarding Time\\n\\nInstant access & 14 days Full Implementation\\n\\n60+ day onboarding plan\\n\\nIntegrations\\n\\nMatlab, Python, PLM Simulation, Excel\\n\\nAnsys, Matlab, some PLM\\n\\nApprox cost (6 editors)\\n\\n$1000/mo\\n\\n$5000/mo\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\nITAR, CUI and Enterprise\\n========================\\n\\n![So Intuitive Engineers Actually Enjoy Using It](https://images.ctfassets.net/c2mtbunjxyfe/53dvorKKwMQJzO2kf8y1iH/0d393173be081259da6755720466c2b4/heart.png?w=96)\\n\\nSo Intuitive Engineers Actually Enjoy Using It\\n\\nTraditional Requirements tools grind engineering to a halt. State of the art interface meets seamless asynchronous communication. Fully onboard your organization within 2 weeks.\\n\\n![Safety, Security, and Privacy](https://images.ctfassets.net/c2mtbunjxyfe/7LJDRDcEYK5NlxxTLYljzt/62b8d575bb5538a2148ff2899dba2dd5/protection.png?w=96)\\n\\nSafety, Security, and Privacy\\n\\nEnterprise-grade security and identity management with SOC 2 compliance. Flow meets ITAR, EAR, CUI, and other global standards. Choose from tailored deployment options, including cloud SaaS, AWS GovCloud, or self-hosted setups.\\n\\n![ITAR](https://images.ctfassets.net/c2mtbunjxyfe/3QY1Hnyxj1YgcugehvmAwc/d510b1319a67021d9ebb31993adbf0c5/ITAR.png?w=256)![GDPR compliant](https://images.ctfassets.net/c2mtbunjxyfe/5SPybkgSGTHqcXvZH9XCaV/9903d801f1aa879f7c687e7b409f1e61/GDPR.png?w=256)![SOC2 compliant](https://images.ctfassets.net/c2mtbunjxyfe/2Rzdp14ru3UDDjCnfT5MTo/ecf5c00258c73558f9aeada00b4d8af7/SOC2.png?w=256)\\n\\n![Flexible to Match Your Existing Process](https://images.ctfassets.net/c2mtbunjxyfe/34THtPoBLT5Rb2VTva4hjy/9b6aebd70e6eadf958cdc58a649dd2c3/puzzle.png?w=96)\\n\\nFlexible to Match Your Existing Process\\n\\nWe're streamlining, not replacing your workflow, so Flow is designed to accommodate your agile team's needs with various configuration options and custom fields.\\n\\nThe default requirements tool for iterative teams\\n=================================================\\n\\nFlow is the agile requirements management solution that’s lightweight enough to speed up your workflows, and rigorous enough to never worry about compliance and traceability again.\\n\\nCatch design misses now and not 90 days from now. Flow helps prevent costly mistakes and get your products to market faster.\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\nAgile Systems Engineering Handbook\\n\\nThis guide is a modern alternative to INCOSE, based on recent, proven experience inside fastest-moving engineering teams. Built in collaboration with the world's fastest teams, it’s designed to support high-speed, real-world engineering needs.\\n\\n[Read the Handbook](https://flowengineering.com/handbook/volume-1/)\\n\\nWebinar: NASA’s Waterfall vs Agile Systems Engineering /w Anduril\\n\\nA webinar with Adam Thurn, Chief Engineer of Space Missions at Anduril, on moving beyond NASA-style waterfall engineering. Real-world agile practices for faster iteration and tighter feedback loops.\\n\\n[Watch Now](https://flowengineering.com/blog/nasa-waterfall-vs-agile-systems-engineering/)\\n\\n[Flow logo](https://flowengineering.com/)\\n\\n© Copyright 2025 TRC Space Ltd.\\n\\nAll rights reserved.\\n\\nProviding new-age engineering companies with a requirements tool that is built specifically for their needs and allows them to focus on engineering ground breaking products.\\n\\n[linkedin](https://www.linkedin.com/company/the-engineering-company)\\n[twitter](https://twitter.com/Flow_Engineer)\\n\\nProduct\\n=======\\n\\n[Pricing](https://flowengineering.com/pricing/)\\n\\n[Release notes](https://flow-engineering.productlane.com/changelog)\\n\\n[Security](https://flowengineering.com/security/)\\n\\n[API](https://api.flowengineering.com/rest/v1/docs)\\n\\nCompany\\n=======\\n\\n[About us](https://flowengineering.com/about-us/)\\n\\n[Careers](https://flowengineering.com/careers/)\\n\\n[Press](https://techcrunch.com/2022/12/06/flow-engineering-wants-to-modernize-the-hardware-engineering-design-process/)\\n\\n[Contact](mailto:«EMAIL»)\\n\\nLegal\\n=====\\n\\n[Disclaimer](https://flowengineering.com/disclaimer/)\\n\\n[Privacy and Cookies](https://flowengineering.com/privacy-and-cookie-policy/)\\n\\nResources\\n=========\\n\\n[Iterative Systems Engineering: In Practice\\\\\\n\\\\\\nNew](https://flowengineering.com/blog/iterative-systems-engineering-in-practice-pdf/)\\n\\n[Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n\\n[Iterative Systems Engineering Handbook](https://flowengineering.com/handbook/)\\n\\n[Agile Systems Engineering at SpaceX](https://flowengineering.com/blog/spacex-systems-engineering-five-tips/)\\n\\n[Functional vs Non-Functional Requirements](https://flowengineering.com/blog/different_types-of_requirements_in_a_hardware_engineering_project/)\\n\\n[Blog](https://flowengineering.com/blog/)\\n\\nFlow raises **$23M Series A** from ![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg)\\n\\n[Read More](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n* [Handbook](https://flowengineering.com/handbook/volume-1/)\\n \\n* [AI](https://flowengineering.com/ai-agent/)\\n \\n* Industries\\n \\n * [Launch Vehicles](https://flowengineering.com/launch-vehicles/)\\n \\n * [Satellites](https://flowengineering.com/satellites/)\\n \\n * [Ground Station](https://flowengineering.com/ground-station/)\\n \\n * [Aviation & Aero](https://flowengineering.com/aviation-and-aero/)\\n \\n * [Fusion](https://flowengineering.com/fusion/)\\n \\n * [Nuclear (SMR)](https://flowengineering.com/nuclear/)\\n \\n * [Quantum](https://flowengineering.com/quantum/)\\n \\n * [eVTOL](https://flowengineering.com/evtol/)\\n \\n * [Automotive](https://flowengineering.com/automotive/)\\n \\n * [Robotics](https://flowengineering.com/robotics/)\\n \\n * [Medical Devices](https://flowengineering.com/medical-devices/)\\n \\n* Product\\n \\n * [Requirements Management](https://flowengineering.com/requirements-management/)\\n \\n * [Architecture](https://flowengineering.com/architecture/)\\n \\n * [Traceability](https://flowengineering.com/traceability/)\\n \\n * [Continuous Verification](https://flowengineering.com/continuous-verification/)\\n \\n * [Live V&V](https://flowengineering.com/live-vv/)\\n \\n * [Test Cases\\\\\\n \\\\\\n new](https://flowengineering.com/test-cases/)\\n \\n * [Regulatory](https://flowengineering.com/regulatory/)\\n \\n * [Integrations](https://flowengineering.com/integrations/)\\n \\n* [Pricing](https://flowengineering.com/pricing/)\\n \\n* Resources\\n \\n * [Blog](https://flowengineering.com/blog/)\\n \\n * [Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n \\n * [Security](https://flowengineering.com/security/)\\n \\n* Company\\n \\n * [About us](https://flowengineering.com/about-us/)\\n \\n * [Careers](https://flowengineering.com/careers/)\\n \\n\\nTest cases\\n==========\\n\\nLightweight Test Cases and Plans. Streamline test orchestration with a clear, visual representation of all test runs and statuses.\\n\\nPowering the planet's fastest-moving hardware teams\\n===================================================\\n\\nPowering the planet's fastest-moving hardware teams\\n\\nWe work with the next generation of hardware companies building complex systems for space, defense, satellites, robotics, new energy nuclear and fusion technologies.\\n\\n![impulse-spaceI](https://images.ctfassets.net/c2mtbunjxyfe/71rebwlPyl8Z54GMToFGuT/7a68d1e327dfc16e8ee102515a0394ae/impulse-space.svg?w=1080)\\n\\n![Joby Aviation](https://images.ctfassets.net/c2mtbunjxyfe/2drKCxYRGI6EYXLmRhmtNc/ca8a72e01bb837728a4eb2cbfdf8e50c/Joby.svg?w=1920)\\n\\n![stoke-logo](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)\\n\\n![astranis-logo](https://images.ctfassets.net/c2mtbunjxyfe/1lG6B3ebgpsYO84jOFJxLt/6d38b98e8991ce21cbf012c1596e4880/Astranis.svg?w=1920)\\n\\n![Radiant](https://images.ctfassets.net/c2mtbunjxyfe/28srfH0gVZQWpTDFjyQ4Xe/055492ff60c71fff494b12a6129982be/Group_661.png?w=1920)\\n\\nBacked by top investors & operators\\n\\nFlow is supported by visionaries driving the new golden age of hardware and by industry leaders who have set the standard for collaboration tools in other fields. [Read more](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg?w=2048)\\n\\nPatrick & John CollisonCEO, Stripe\\n\\nKyle ParrishFormer VP Sales, Figma\\n\\nDavid Helgason Former CEO, Unity\\n\\nThe default requirements tool for iterative teams\\n=================================================\\n\\nFlow is the agile requirements management solution that’s lightweight enough to speed up your workflows, and rigorous enough to never worry about compliance and traceability again.\\n\\nCatch design misses now and not 90 days from now. Flow helps prevent costly mistakes and get your products to market faster.\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\n[Flow logo](https://flowengineering.com/)\\n\\n© Copyright 2025 TRC Space Ltd.\\n\\nAll rights reserved.\\n\\nProviding new-age engineering companies with a requirements tool that is built specifically for their needs and allows them to focus on engineering ground breaking products.\\n\\n[linkedin](https://www.linkedin.com/company/the-engineering-company)\\n[twitter](https://twitter.com/Flow_Engineer)\\n\\nProduct\\n=======\\n\\n[Pricing](https://flowengineering.com/pricing/)\\n\\n[Release notes](https://flow-engineering.productlane.com/changelog)\\n\\n[Security](https://flowengineering.com/security/)\\n\\n[API](https://api.flowengineering.com/rest/v1/docs)\\n\\nCompany\\n=======\\n\\n[About us](https://flowengineering.com/about-us/)\\n\\n[Careers](https://flowengineering.com/careers/)\\n\\n[Press](https://techcrunch.com/2022/12/06/flow-engineering-wants-to-modernize-the-hardware-engineering-design-process/)\\n\\n[Contact](mailto:«EMAIL»)\\n\\nLegal\\n=====\\n\\n[Disclaimer](https://flowengineering.com/disclaimer/)\\n\\n[Privacy and Cookies](https://flowengineering.com/privacy-and-cookie-policy/)\\n\\nResources\\n=========\\n\\n[Iterative Systems Engineering: In Practice\\\\\\n\\\\\\nNew](https://flowengineering.com/blog/iterative-systems-engineering-in-practice-pdf/)\\n\\n[Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n\\n[Iterative Systems Engineering Handbook](https://flowengineering.com/handbook/)\\n\\n[Agile Systems Engineering at SpaceX](https://flowengineering.com/blog/spacex-systems-engineering-five-tips/)\\n\\n[Functional vs Non-Functional Requirements](https://flowengineering.com/blog/different_types-of_requirements_in_a_hardware_engineering_project/)\\n\\n[Blog](https://flowengineering.com/blog/)\" }", "type": "array" } ] }, "options": {} }, "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 176, 0 ], "id": "a3db76d8-b305-40f6-9ed9-63090dbddad7", "name": "Case Study" }, { "parameters": { "assignments": { "assignments": [ { "id": "b5d5b4c7-516f-42b3-b83f-d203f3744ed2", "name": "ICP", "value": "[ { \"merged_markdown\": \"Flow raises **$23M Series A** from ![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg)\\n\\n[Read More](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n* [Handbook](https://flowengineering.com/handbook/volume-1/)\\n \\n* [AI](https://flowengineering.com/ai-agent/)\\n \\n* Industries\\n \\n * [Launch Vehicles](https://flowengineering.com/launch-vehicles/)\\n \\n * [Satellites](https://flowengineering.com/satellites/)\\n \\n * [Ground Station](https://flowengineering.com/ground-station/)\\n \\n * [Aviation & Aero](https://flowengineering.com/aviation-and-aero/)\\n \\n * [Fusion](https://flowengineering.com/fusion/)\\n \\n * [Nuclear (SMR)](https://flowengineering.com/nuclear/)\\n \\n * [Quantum](https://flowengineering.com/quantum/)\\n \\n * [eVTOL](https://flowengineering.com/evtol/)\\n \\n * [Automotive](https://flowengineering.com/automotive/)\\n \\n * [Robotics](https://flowengineering.com/robotics/)\\n \\n * [Medical Devices](https://flowengineering.com/medical-devices/)\\n \\n* Product\\n \\n * [Requirements Management](https://flowengineering.com/requirements-management/)\\n \\n * [Architecture](https://flowengineering.com/architecture/)\\n \\n * [Traceability](https://flowengineering.com/traceability/)\\n \\n * [Continuous Verification](https://flowengineering.com/continuous-verification/)\\n \\n * [Live V&V](https://flowengineering.com/live-vv/)\\n \\n * [Test Cases\\\\\\n \\\\\\n new](https://flowengineering.com/test-cases/)\\n \\n * [Regulatory](https://flowengineering.com/regulatory/)\\n \\n * [Integrations](https://flowengineering.com/integrations/)\\n \\n* [Pricing](https://flowengineering.com/pricing/)\\n \\n* Resources\\n \\n * [Blog](https://flowengineering.com/blog/)\\n \\n * [Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n \\n * [Security](https://flowengineering.com/security/)\\n \\n* Company\\n \\n * [About us](https://flowengineering.com/about-us/)\\n \\n * [Careers](https://flowengineering.com/careers/)\\n \\n\\nRequirements tool built for rapid development\\n=============================================\\n\\nThe first tool custom built for iterative, cross-functional engineering teams to design & iterate faster\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\n[![Mach Industries](https://images.ctfassets.net/c2mtbunjxyfe/50nWLJRL9wtu5NVk8EqFDc/e351c326e2410330a78c91c63081d583/5b615526-0071-4c7b-96af-11672b4a124a.svg?w=256)](https://machindustries.co/)\\n[![Radiant](https://images.ctfassets.net/c2mtbunjxyfe/6OiITxBbNx2mEabwnUAtlK/5d0a47c9527eb718432fc3c133c0903c/Radiant.png?w=1920)](https://www.radiantnuclear.com/)\\n[![Joby](https://images.ctfassets.net/c2mtbunjxyfe/4cvUNWFbixz8L6FQGCfuck/5ff7b7a2684d84e40162b828bd453d72/Joby.svg?w=1920)](https://www.jobyaviation.com/)\\n[![Rivian](https://images.ctfassets.net/c2mtbunjxyfe/VOLksYa01iqJIBnYtTwar/86203dce1085fa6ae8a3b3f16039bc2b/rivian-logo.svg?w=1080)](https://rivian.com/)\\n[![Astranis](https://images.ctfassets.net/c2mtbunjxyfe/5ZZHV79zARsLZkorcUaA2H/a1dcb0f27bc8dc155e60fd95417e1a0d/Astranis.svg?w=1920)](https://www.astranis.com/)\\n[![Impulse Space](https://images.ctfassets.net/c2mtbunjxyfe/71rebwlPyl8Z54GMToFGuT/7a68d1e327dfc16e8ee102515a0394ae/impulse-space.svg?w=1080)](https://www.impulsespace.com/)\\n[![Stoke](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)](https://www.stokespace.com/)\\n\\n![background-flow-requirements](https://images.ctfassets.net/c2mtbunjxyfe/4LI2XPWn5Vpst3k4hE90v/7061a049ea2c51554243bdd4e750bd02/Group_201.jpg?w=3840)![background-flow-systems](https://images.ctfassets.net/c2mtbunjxyfe/18L2uQ5YXQ0jSN8Jj9N205/e40ad32fe1d79b3c7a79052b5b0e142e/Group_203.jpg?w=3840)![background-flow-architecture](https://images.ctfassets.net/c2mtbunjxyfe/6Ngm6MDAw5sPBZSpwv81x8/bdb70e16d9850b21e155a15dc6cc8f10/Group_200.jpg?w=3840)![background-flow-live-v-and-v](https://images.ctfassets.net/c2mtbunjxyfe/1apz8Vk8WhQptrrI6UnBP5/8e4bd187830042229ff0860ba6f16c28/Group_202.jpg?w=3840)![background-test-cases](https://images.ctfassets.net/c2mtbunjxyfe/4cdwdvMPjh8MSo7XT97SLM/de4ca9c49fc9a209dad9cc9409ce7aa5/background-test-cases.jpg?w=3840)![background-flow-requirements](https://images.ctfassets.net/c2mtbunjxyfe/49DeRgSdpfyUKG7b6gyWmA/731fad76a91d4aabf133fe8ad344d3e1/Group_201.jpg?w=3840)\\n\\nEnd-to-end traceability of requirements & design parameters from design to release\\n\\nEasy for non-systems engineers to collaborate seamlessly\\n\\nA unified view of your systems and interfaces that everyone can use.\\n\\nSee live status of requirements with integrations to design and test data. CI/CD for systems engineering.\\n\\nStreamline test orchestration with a clear, visual representation of all test runs and statuses.\\n\\nFlow is an AI native systems platform. Our agent works on top of your most important design & test data so that you speed up critical work\\n\\n![placeholder Requirements](https://images.ctfassets.net/c2mtbunjxyfe/5Jk2nVPSmbyKzhNnViTBQt/abcfbdc47b11eec402c8535aca96fc2d/ezgif-frame-001.jpg?w=3840)\\n\\n![placeholder Systems](https://images.ctfassets.net/c2mtbunjxyfe/3ltfqP2armZBLwuCyHG2Ci/0ac69a7d215781817bb5441b89f85756/ezgif-frame-001__2_.jpg?w=3840)\\n\\n![placeholder Architecture](https://images.ctfassets.net/c2mtbunjxyfe/109KeEQAybmHEuXsWmLeKO/c72275973bd3a52793136f0b77dbd607/ezgif-frame-001__3_.jpg?w=3840)\\n\\n![placeholder Integrations](https://images.ctfassets.net/c2mtbunjxyfe/1xwp3k67x2msEXStKYMddE/b4d2fdeef6cba0aeb2c05580b02276a7/ezgif-frame-001__4_.jpg?w=3840)\\n\\n![placeholder Test Cases](https://images.ctfassets.net/c2mtbunjxyfe/35UwA6i6nbtWYWpKFn7NZW/b2c791f525302300282493c2ba6ed20a/Screenshot_2025-01-09_at_17.59.29.png?w=3840)\\n\\n![placeholder AI Agent](https://images.ctfassets.net/c2mtbunjxyfe/3eaPVEwdd7BS2TjmGjkbaT/b4b9eb1ce4d83f03717001dd1f1c4030/ezgif-frame-001__1_.jpg?w=3840)\\n\\nBuilt for Iterative Hardware teams\\n\\n[![Rivian](https://images.ctfassets.net/c2mtbunjxyfe/VOLksYa01iqJIBnYtTwar/86203dce1085fa6ae8a3b3f16039bc2b/rivian-logo.svg?w=1080)](https://rivian.com/)\\n[![Joby](https://images.ctfassets.net/c2mtbunjxyfe/2drKCxYRGI6EYXLmRhmtNc/ca8a72e01bb837728a4eb2cbfdf8e50c/Joby.svg?w=1920)](https://www.jobyaviation.com/)\\n[![Astranis](https://images.ctfassets.net/c2mtbunjxyfe/1lG6B3ebgpsYO84jOFJxLt/6d38b98e8991ce21cbf012c1596e4880/Astranis.svg?w=1920)](https://www.astranis.com/)\\n[![Impulse Space](https://images.ctfassets.net/c2mtbunjxyfe/71rebwlPyl8Z54GMToFGuT/7a68d1e327dfc16e8ee102515a0394ae/impulse-space.svg?w=1080)](https://www.impulsespace.com/)\\n[![Stoke](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)](https://www.stokespace.com/)\\n[![Radiant Nuclear](https://images.ctfassets.net/c2mtbunjxyfe/28srfH0gVZQWpTDFjyQ4Xe/055492ff60c71fff494b12a6129982be/Group_661.png?w=1920)](https://www.radiantnuclear.com/)\\n[![Skydio](https://images.ctfassets.net/c2mtbunjxyfe/1diwTaRqyYNrpjgr6jENLK/410f01fc04ab515538ad87a93fef638e/skydio.svg?w=1080)](https://www.skydio.com/)\\n[![Mach Industries](https://images.ctfassets.net/c2mtbunjxyfe/50nWLJRL9wtu5NVk8EqFDc/e351c326e2410330a78c91c63081d583/5b615526-0071-4c7b-96af-11672b4a124a.svg?w=256)](https://machindustries.co/)\\n[![Firehawk Aerospace](https://images.ctfassets.net/c2mtbunjxyfe/SqdXXZevmgohRAwexIPKX/bce0d7192b9eab97acea35eff7516696/firehawk.svg?w=1080)](https://firehawkaerospace.com/)\\n[![Antares Industries](https://images.ctfassets.net/c2mtbunjxyfe/1gzMZ0lvyjnwOLo9aLNO1g/94271e6f85a5748159645906203cca24/Group_659.png?w=1200)](https://antaresindustries.com/)\\n[![Aalyria](https://images.ctfassets.net/c2mtbunjxyfe/4RkdCzDNqGjfqDtf7AHVnf/56b0bc19066fe06cfc0c9ca02f683dc0/Aalyria.svg?w=1920)](https://www.aalyria.com/)\\n[![Harbinger Motor](https://images.ctfassets.net/c2mtbunjxyfe/7hl1ezrBNAfQlNLNe7r413/345d53ce65370c037bb1395eada99bd7/harbinger-logo_02_2.svg?w=1200)](https://harbingermotors.com/)\\n[![Paebbl](https://images.ctfassets.net/c2mtbunjxyfe/IPcWhM1mDxCNeIQ0eLOxJ/91bb611b159c30a8b86b5aaacbd2d334/Paebbl_Lockup_CB_1.svg?w=1080)](https://paebbl.com/)\\n[![Hyphen](https://images.ctfassets.net/c2mtbunjxyfe/3Pc2fRHjFBduWqgJSQ4iTz/6352754e588dd28a10cec60cabc547a5/Hyphen_Logo.svg?w=3840)](https://usehyphen.com/)\\n[![Gravitics](https://images.ctfassets.net/c2mtbunjxyfe/1MBeMlKw9cNX5uIpVTl9py/f5d2d8ec439d2ac5aef97849ba547bb6/gravitics.svg?w=1920)](https://www.gravitics.com/)\\n[![Space Machines Company](https://images.ctfassets.net/c2mtbunjxyfe/1vRV1ZuPkHw17jvS2mkrhT/e8a503d56f94606d692e9d9230df1108/spacemachines.svg?w=1080)](https://www.spacemachines.co/)\\n[![Turion Space](https://images.ctfassets.net/c2mtbunjxyfe/644448yAD8DDlYB1CGXmAH/9cf2937db8adda33387faa34bd150622/Group_662.png?w=1920)](https://turionspace.com/#contact)\\n[![Openstar](https://images.ctfassets.net/c2mtbunjxyfe/1y4lEFfq5rKoKfQI7GsBJD/06a519aa84916ce60673fd74ef3d6e9d/Openstar.svg?w=1920)](https://www.openstar.tech/)\\n[![Phase Four](https://images.ctfassets.net/c2mtbunjxyfe/5OVBKR4fwGwRFvMUSzDjgs/6cd1252895bf4a725a21ef84ecbc5bc3/phasefour.svg?w=1080)](https://phasefour.io/)\\n[![Intramotev](https://images.ctfassets.net/c2mtbunjxyfe/38IKMTgonUFuKTcdAY8bNA/2a0d2ce31dba8850f25aeb196aca8147/Frame_15.png?w=1920)](https://intramotev.com/)\\n\\n**Iterate faster**. Integrate your design tools with requirements for live V&V tracking.\\n========================================================================================\\n\\nCollaborateParameterizeIntegrateVerifyRelease\\n\\n#### **Collaborate**. Give engineers real ownership of requirements. Help everyone take a system first mindset.\\n\\n![collaborative](https://images.ctfassets.net/c2mtbunjxyfe/6f30mkFgsXMwsTqPEbfEbN/24794e454d49c56031490bf1bc1e892a/collaborative.jpg?w=3840)\\n\\n#### **Parameterize**. Convert text-based requirements into machine readable checks for budgets or actuals.\\n\\n![parametrise](https://images.ctfassets.net/c2mtbunjxyfe/7yASuu0i2X3Pb1uULZypux/1f6cb3f6444b333223f86796849aa7c7/parametrise.jpg?w=3840)\\n\\n#### **Integrate**. Drive parameters via API's or integrations for live V&V tracking.\\n\\n![integrate](https://images.ctfassets.net/c2mtbunjxyfe/1K50IkC6c3Prc2gHw2Kfkw/e4076b0e6ca32498df5b9b8d96d68d77/integrate.jpg?w=3840)\\n\\n#### **Verify**. Continuously verify your design on every major design change saving weeks of manual checks.\\n\\n![verify](https://images.ctfassets.net/c2mtbunjxyfe/3JZTjJQhRsUtJqNMvVYLPp/c23009a771f80bcf50bc8fdae94c9ca7/verify.jpg?w=3840)\\n\\n#### **Release**. Generate reports for regulators with end to end design traceability.\\n\\n![release](https://images.ctfassets.net/c2mtbunjxyfe/3bVRNAWQAHVtqIxr6MTMRD/d21e8c4c231b58edd3437335699db157/release.jpg?w=3840)\\n\\nIntegrate, propagate and manage change in minutes\\n=================================================\\n\\nWithout FlowWith Flow\\n\\nNo clean way to communicate requirements\\n\\nEveryone is out of sync\\n\\nFragile, unscalable and error prone spreadsheets\\n\\nEngineering team working in isolation\\n\\nDifficult to manage and see interdependencies\\n\\n![without flow](https://images.ctfassets.net/c2mtbunjxyfe/4t9nq21if0SU0uv0TAokp1/699af5cdefdae9372d6aba725e2a0674/without-flow-export.png?w=3840)\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/3X7wEu4hTIANoO6j5DIAuu/03d0da2301d035b8c0d0f2cb81c34d2b/stoke_1.svg?w=1920)\\n\\nFlow is the requirements management tool of our dreams\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nFlow is the requirements management tool of our dreams. It reflects how we work—fast, iterative, and focused on giving everyone a common, integrated view of our system.\\n\\nGetting to CI/CD for systems is the dream. Flow is what finally got us there. It’s not just another tool, it’s the backbone of how we build now.\\n\\n**We’ve had new hires tell us, unprompted, ‘this is the best requirements tool I’ve ever used’.** That says it all.\\n\\n![Ryan McCullough](https://images.ctfassets.net/c2mtbunjxyfe/BwghBr8T2Rac5c8Bp4lll/954ef86b58d1559d131a40ce62517fa2/1674681902140.jpeg?w=96)\\n\\nRyan McCullough\\n\\nSystem Architect\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/4RkdCzDNqGjfqDtf7AHVnf/56b0bc19066fe06cfc0c9ca02f683dc0/Aalyria.svg?w=1920)\\n\\nThere’s nothing on the market that compares\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nWe needed a tool that could keep up with the pace of our organization, adapt to the increasing complexity of our products, and remain easy to manage without full time dedicated resources. Flow has delivered.\\n\\nUnlike many tools that disrupt workflows or create bottlenecks, **Flow integrates successfully with how modern engineering teams operate.** It’s fast, flexible, and scales with ease.\\n\\nIn my experience, there’s nothing on the market that compares.\\n\\n![Joe Barton](https://images.ctfassets.net/c2mtbunjxyfe/fWufVVTUhdrZ9s933sdlr/ddbafed64eed1d1624b8910f1591af7f/1563737889020.jpeg?w=96)\\n\\nJoe Barton\\n\\nTechnical Program Manager\\n\\n![](https://images.ctfassets.net/c2mtbunjxyfe/1vRV1ZuPkHw17jvS2mkrhT/e8a503d56f94606d692e9d9230df1108/spacemachines.svg?w=1080)\\n\\nI have not found anything comparable to Flow in giving us flexibility without compromising functionality\\n\\n![Quote mark](https://flowengineering.com/_next/static/chunks/svg/quotes-f3b3defc0f90daaad676cc6c976015d8.svg?w=96)\\n\\nBefore Flow, this meant juggling spreadsheets, duplicate docs, and months of rework. Now it just works. I have not found anything comparable to Flow in giving us **flexibility without compromising functionality**.\\n\\nWith Flow, all our requirements, design targets, and simulation data actually live in one place. That might not sound revolutionary—but if you’ve ever tried to do this in hardware, you know how painful it actually is.\\n\\n![Rajat](https://images.ctfassets.net/c2mtbunjxyfe/6OyqfjAtOwDESMXXfyFUVT/ba34a3c74a936e63bd1314fab803cb76/1516229306173.jpeg?w=96)\\n\\nRajat Kulshrestha\\n\\nCEO & Co-Founder, Space Machines Company\\n\\n1 / 3\\n\\nBuilt for cross-functional teams\\n\\nWhere iterative teams meet regulatory compliance\\n================================================\\n\\nSystem Engineers\\n\\nChief Engineers\\n\\nResponsible Engineers\\n\\nSystem Engineers | Levels\\n=========================\\n\\nSpend less time chasing down requirements updates from your engineers and spend more time optimizing your system to build better products.\\n\\n* Save dozens of hours for each design review\\n* Automated verification from your team & tools\\n* Identify problems before it's too late\\n\\n![System Engineers | Levels](https://images.ctfassets.net/c2mtbunjxyfe/6B4WqtlLlkzknpLJYOXFo8/653b810b3df2a83afb5078bf24a0c3b0/Property_1_Default.png?w=3840)\\n\\nFlexible but powerful\\n=====================\\n\\nReplace the hassle of manually updating spreadsheets or using cumbersome enterprise tools by upgrading your workflow with collaborative requirements management that's always up to date, for everyone.\\n\\nAgile and scalable\\n\\n![Flow](https://images.ctfassets.net/c2mtbunjxyfe/2Zqj08RbDH0XRgIfZYdky3/a936fdee09982e7d1c537b58cb07411c/Vector.png?w=48)Flow\\n\\nExpensive legacy tools\\n\\n![Enterprise](https://images.ctfassets.net/c2mtbunjxyfe/4Ao6Bzcbn3Q1qYe7bbARwa/a6232f8f08f2ea67c25d0b9fcac7d105/ibm-rational-doors_215495.webp?w=48)IBM Doors\\n\\nITAR Compliance\\n\\nContinuous Verification\\n\\nBaselines, config, history\\n\\nAccessible & Collaborative\\n\\nMBSE & Architecture\\n\\nFast, Flexible & Easy\\n\\nCustomization\\n\\nOnboarding Time\\n\\nInstant access & 14 days Full Implementation\\n\\n60+ day onboarding plan\\n\\nIntegrations\\n\\nMatlab, Python, PLM Simulation, Excel\\n\\nAnsys, Matlab, some PLM\\n\\nApprox cost (6 editors)\\n\\n$1000/mo\\n\\n$5000/mo\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\nITAR, CUI and Enterprise\\n========================\\n\\n![So Intuitive Engineers Actually Enjoy Using It](https://images.ctfassets.net/c2mtbunjxyfe/53dvorKKwMQJzO2kf8y1iH/0d393173be081259da6755720466c2b4/heart.png?w=96)\\n\\nSo Intuitive Engineers Actually Enjoy Using It\\n\\nTraditional Requirements tools grind engineering to a halt. State of the art interface meets seamless asynchronous communication. Fully onboard your organization within 2 weeks.\\n\\n![Safety, Security, and Privacy](https://images.ctfassets.net/c2mtbunjxyfe/7LJDRDcEYK5NlxxTLYljzt/62b8d575bb5538a2148ff2899dba2dd5/protection.png?w=96)\\n\\nSafety, Security, and Privacy\\n\\nEnterprise-grade security and identity management with SOC 2 compliance. Flow meets ITAR, EAR, CUI, and other global standards. Choose from tailored deployment options, including cloud SaaS, AWS GovCloud, or self-hosted setups.\\n\\n![ITAR](https://images.ctfassets.net/c2mtbunjxyfe/3QY1Hnyxj1YgcugehvmAwc/d510b1319a67021d9ebb31993adbf0c5/ITAR.png?w=256)![GDPR compliant](https://images.ctfassets.net/c2mtbunjxyfe/5SPybkgSGTHqcXvZH9XCaV/9903d801f1aa879f7c687e7b409f1e61/GDPR.png?w=256)![SOC2 compliant](https://images.ctfassets.net/c2mtbunjxyfe/2Rzdp14ru3UDDjCnfT5MTo/ecf5c00258c73558f9aeada00b4d8af7/SOC2.png?w=256)\\n\\n![Flexible to Match Your Existing Process](https://images.ctfassets.net/c2mtbunjxyfe/34THtPoBLT5Rb2VTva4hjy/9b6aebd70e6eadf958cdc58a649dd2c3/puzzle.png?w=96)\\n\\nFlexible to Match Your Existing Process\\n\\nWe're streamlining, not replacing your workflow, so Flow is designed to accommodate your agile team's needs with various configuration options and custom fields.\\n\\nThe default requirements tool for iterative teams\\n=================================================\\n\\nFlow is the agile requirements management solution that’s lightweight enough to speed up your workflows, and rigorous enough to never worry about compliance and traceability again.\\n\\nCatch design misses now and not 90 days from now. Flow helps prevent costly mistakes and get your products to market faster.\\n\\n* [Book a Demo](https://flowengineering.com/demo/)\\n \\n* [Start Free Trial](https://app.flowengineering.com/free-trial)\\n \\n\\nAgile Systems Engineering Handbook\\n\\nThis guide is a modern alternative to INCOSE, based on recent, proven experience inside fastest-moving engineering teams. Built in collaboration with the world's fastest teams, it’s designed to support high-speed, real-world engineering needs.\\n\\n[Read the Handbook](https://flowengineering.com/handbook/volume-1/)\\n\\nWebinar: NASA’s Waterfall vs Agile Systems Engineering /w Anduril\\n\\nA webinar with Adam Thurn, Chief Engineer of Space Missions at Anduril, on moving beyond NASA-style waterfall engineering. Real-world agile practices for faster iteration and tighter feedback loops.\\n\\n[Watch Now](https://flowengineering.com/blog/nasa-waterfall-vs-agile-systems-engineering/)\\n\\n[Flow logo](https://flowengineering.com/)\\n\\n© Copyright 2025 TRC Space Ltd.\\n\\nAll rights reserved.\\n\\nProviding new-age engineering companies with a requirements tool that is built specifically for their needs and allows them to focus on engineering ground breaking products.\\n\\n[linkedin](https://www.linkedin.com/company/the-engineering-company)\\n[twitter](https://twitter.com/Flow_Engineer)\\n\\nProduct\\n=======\\n\\n[Pricing](https://flowengineering.com/pricing/)\\n\\n[Release notes](https://flow-engineering.productlane.com/changelog)\\n\\n[Security](https://flowengineering.com/security/)\\n\\n[API](https://api.flowengineering.com/rest/v1/docs)\\n\\nCompany\\n=======\\n\\n[About us](https://flowengineering.com/about-us/)\\n\\n[Careers](https://flowengineering.com/careers/)\\n\\n[Press](https://techcrunch.com/2022/12/06/flow-engineering-wants-to-modernize-the-hardware-engineering-design-process/)\\n\\n[Contact](mailto:«EMAIL»)\\n\\nLegal\\n=====\\n\\n[Disclaimer](https://flowengineering.com/disclaimer/)\\n\\n[Privacy and Cookies](https://flowengineering.com/privacy-and-cookie-policy/)\\n\\nResources\\n=========\\n\\n[Iterative Systems Engineering: In Practice\\\\\\n\\\\\\nNew](https://flowengineering.com/blog/iterative-systems-engineering-in-practice-pdf/)\\n\\n[Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n\\n[Iterative Systems Engineering Handbook](https://flowengineering.com/handbook/)\\n\\n[Agile Systems Engineering at SpaceX](https://flowengineering.com/blog/spacex-systems-engineering-five-tips/)\\n\\n[Functional vs Non-Functional Requirements](https://flowengineering.com/blog/different_types-of_requirements_in_a_hardware_engineering_project/)\\n\\n[Blog](https://flowengineering.com/blog/)\\n\\nFlow raises **$23M Series A** from ![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg)\\n\\n[Read More](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n* [Handbook](https://flowengineering.com/handbook/volume-1/)\\n \\n* [AI](https://flowengineering.com/ai-agent/)\\n \\n* Industries\\n \\n * [Launch Vehicles](https://flowengineering.com/launch-vehicles/)\\n \\n * [Satellites](https://flowengineering.com/satellites/)\\n \\n * [Ground Station](https://flowengineering.com/ground-station/)\\n \\n * [Aviation & Aero](https://flowengineering.com/aviation-and-aero/)\\n \\n * [Fusion](https://flowengineering.com/fusion/)\\n \\n * [Nuclear (SMR)](https://flowengineering.com/nuclear/)\\n \\n * [Quantum](https://flowengineering.com/quantum/)\\n \\n * [eVTOL](https://flowengineering.com/evtol/)\\n \\n * [Automotive](https://flowengineering.com/automotive/)\\n \\n * [Robotics](https://flowengineering.com/robotics/)\\n \\n * [Medical Devices](https://flowengineering.com/medical-devices/)\\n \\n* Product\\n \\n * [Requirements Management](https://flowengineering.com/requirements-management/)\\n \\n * [Architecture](https://flowengineering.com/architecture/)\\n \\n * [Traceability](https://flowengineering.com/traceability/)\\n \\n * [Continuous Verification](https://flowengineering.com/continuous-verification/)\\n \\n * [Live V&V](https://flowengineering.com/live-vv/)\\n \\n * [Test Cases\\\\\\n \\\\\\n new](https://flowengineering.com/test-cases/)\\n \\n * [Regulatory](https://flowengineering.com/regulatory/)\\n \\n * [Integrations](https://flowengineering.com/integrations/)\\n \\n* [Pricing](https://flowengineering.com/pricing/)\\n \\n* Resources\\n \\n * [Blog](https://flowengineering.com/blog/)\\n \\n * [Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n \\n * [Security](https://flowengineering.com/security/)\\n \\n* Company\\n \\n * [About us](https://flowengineering.com/about-us/)\\n \\n * [Careers](https://flowengineering.com/careers/)\\n \\n\\nIntegrations\\n============\\n\\nExplore Flow’s growing ecosystem of integrations. Connect your design, simulation, and collaboration tools seamlessly.\\n\\n![Python](https://images.ctfassets.net/c2mtbunjxyfe/1Te5cEYCQBW6WOuAzuElxu/2c94e8839d94c6a6091723bced7a7825/python.svg?w=640)\\n\\n##### Python\\n\\nSeamlessly connect your Python scripts to Flow so analysis results stay tightly linked to requirements.\\n\\n[![Github](https://images.ctfassets.net/c2mtbunjxyfe/2Pvu4yh8or4PFZ3l4GNgWi/23081179a145fbfe5798ed1e903a29da/github.svg?w=640)\\\\\\n\\\\\\n##### Github\\\\\\n\\\\\\nAutomatically extract data from calculation scripts and CSVs stored under source control](https://flowengineering.com/github/)\\n\\n![Siemens NX](https://images.ctfassets.net/c2mtbunjxyfe/5YPEsgWyZrWCl9XdIjEZdT/c6c0b8f2bcb0c275c72f032fb010ee1f/nx.svg?w=640)\\n\\n##### Siemens NX\\n\\ncoming soon\\n\\nLink Siemens NX models and data to Flow so design elements remain aligned with your evolving requirements.\\n\\n[![API](https://images.ctfassets.net/c2mtbunjxyfe/6DFvCjSN6gInuywoluj2q5/0aad1fbb4b6d1f0538b759190ee44ba9/mOF8h9FLaVeQN5YtORL3iAJ0.png?w=1080)\\\\\\n\\\\\\n##### API\\\\\\n\\\\\\nManage and update requirements in Flow for seamless data integration with your existing tools.](https://api.flowengineering.com/rest/v1/docs)\\n\\n![Ansys](https://images.ctfassets.net/c2mtbunjxyfe/43iebCIshG6hwRz0UHZ7zD/81c71d81bc02303d557d471a72f457c1/ansys.svg?w=640)\\n\\n##### Ansys\\n\\nComing Soon\\n\\nIntegrate simulation outputs to keep performance insights directly tied to requirements.\\n\\n[![Matlab](https://images.ctfassets.net/c2mtbunjxyfe/7MDqcVZs7ywc8d8pEF7YEV/99b06ca22fe7f12ad30238827bc3f0a1/matlab.png?w=640)\\\\\\n\\\\\\n##### Matlab\\\\\\n\\\\\\nExtract results from each script run and automatically verify those results against requirements.](https://flowengineering.com/matlab/)\\n[![Google Sheets](https://images.ctfassets.net/c2mtbunjxyfe/5mDv1QdNqOYDaxkH8rvDA0/2a7ede050a854fb47fcdea78d5cb7094/google_sheets.svg?w=640)\\\\\\n\\\\\\n##### Google Sheets\\\\\\n\\\\\\nDerive requirement and drive requirements verification using native spreadsheet integration](https://flowengineering.com/google-sheets/)\\n[![Excel](https://images.ctfassets.net/c2mtbunjxyfe/LkWAH0J8udChxkheBpFIt/64dc10cf4d43ed6b4e9aea1ce98ebf21/Microsoft_Office_Excel__2019_present_.svg_1.png?w=256)\\\\\\n\\\\\\n##### Excel\\\\\\n\\\\\\nConnect spreadsheets, maintain versioning and extract data from specific cells and check those values against your requirements.](https://flowengineering.com/excel/)\\n[![Simscale](https://images.ctfassets.net/c2mtbunjxyfe/dEciYr7i41LroXMZR4re7/42539facf3c9a537bc4a85dded49fb58/simscale.svg?w=640)\\\\\\n\\\\\\n##### Simscale\\\\\\n\\\\\\nVerify component-level requirements by linking SimScale simulation results for instant pass/fail feedback.](https://flowengineering.com/simscale/)\\n[![Jira](https://images.ctfassets.net/c2mtbunjxyfe/6rpGLjgBjwtzwJTToKx9xE/823434237348e5a7e3cfb0da05f5acca/jira.svg?w=640)\\\\\\n\\\\\\n##### Jira\\\\\\n\\\\\\nRichly and dynamically link tickets from Jira to your requirements and see real-time status](https://flowengineering.com/jira/)\\n[![Onshape](https://images.ctfassets.net/c2mtbunjxyfe/1yAY7JIBNPQq688GwSdcbi/ca62000ae423cb574b307a3959313d69/onshape.svg?w=640)\\\\\\n\\\\\\n##### Onshape\\\\\\n\\\\\\nPull critical design data such as lengths, masses and volumes directly from your Onshape model for automated verification.](https://flowengineering.com/onshape/)\\n\\n![Solidworks](https://images.ctfassets.net/c2mtbunjxyfe/2CqVKX3vb35Mmg8xqaftGS/cf6773dcc132455152523f18d0462ee8/solidworks.svg?w=640)\\n\\n##### Solidworks\\n\\ncoming soon\\n\\nTie SolidWorks design data to Flow requirements, ensuring your engineering updates are always requirements-driven.\\n\\n![Altium 365](https://images.ctfassets.net/c2mtbunjxyfe/2C2FVUr2CUIyd8IPfgdrhX/88971c85ccbdbb9186c35fe991e8a47b/altium.svg?w=640)\\n\\n##### Altium 365\\n\\ncoming soon\\n\\nUnite PCB design details from Altium 365 with Flow’s requirements so hardware specs stay in perfect sync.\\n\\n![GitLab](https://images.ctfassets.net/c2mtbunjxyfe/2O160czhisTIFpUY359BrQ/233b9584fbb16862fe5a37ff09b80e95/gitlab.svg?w=3840)\\n\\n##### GitLab\\n\\nComing soon\\n\\nLink Flow’s requirements to GitLab commits for real-time code tracking and streamlined collaboration.\\n\\n![Fusion by Stoke](https://images.ctfassets.net/c2mtbunjxyfe/57MLRFObkvEItJmGURuPuA/f4b4c132796e248c1d59368775b79c61/fusion-by-stoke.svg?w=640)\\n\\n##### Fusion by Stoke\\n\\ncoming soon\\n\\nUnify Flow’s requirements with Fusion by Stoke to align manufacturing data and product specifications effortlessly.\\n\\n![Fusion 360](https://images.ctfassets.net/c2mtbunjxyfe/1cIC8JLGqLK2CWao88twyt/c757107d8c5d0dba3eb0c6593b6153cd/fusion360.svg?w=640)\\n\\n##### Fusion 360\\n\\nKeep 3D models and manufacturing data from Fusion 360 aligned with Flow’s requirements in real time.\\n\\n[![Epsilon 3](https://images.ctfassets.net/c2mtbunjxyfe/6Opi5ulUf7iyvJYBzUxowU/efe5bdad3448d20b33ec38abd02de02f/epsilon_3.svg?w=640)\\\\\\n\\\\\\n##### Epsilon 3\\\\\\n\\\\\\nBring Epsilon 3 test plans into Flow to unify test procedures and requirements in one collaborative platform.](https://flowengineering.com/epsilon3/)\\n[![Duro](https://images.ctfassets.net/c2mtbunjxyfe/77hh1b3FAIwa0AdnAZjI0m/a8af61087619715ca3c2e5dea4931fe6/Frame_212.svg?w=128)\\\\\\n\\\\\\n##### Duro\\\\\\n\\\\\\nComing Soon\\\\\\n\\\\\\nKeep design changes fully traceable by syncing Duro’s PLM data with Flow’s requirements in real time.](https://flowengineering.com/duro/)\\n[![First Resonance ION](https://images.ctfassets.net/c2mtbunjxyfe/22FMmeeyXxRZgaYPX9x78H/72f794cb585243be95a356fd8b6881e5/first-resonance.svg?w=640)\\\\\\n\\\\\\n##### First Resonance ION\\\\\\n\\\\\\nLink component-level requirements to ION for tracking and receiving instant feedback from simulations.](https://flowengineering.com/first-resonance/)\\n\\n![Manufacturo](https://images.ctfassets.net/c2mtbunjxyfe/2qajIJ4g4qaRbGgccU8n3q/5bff3c49120c8fc46bde76b2e24f71a9/manufacturo.svg?w=640)\\n\\n##### Manufacturo\\n\\ncoming soon\\n\\nSync production metrics from Manufacturo to Flow so shop-floor feedback directly informs requirement fulfillment.\\n\\nFrequently asked questions\\n==========================\\n\\nEverything you need to know about integrations.\\n\\nDo you have an API?\\n\\nWe have a flexible REST API. You can read more in our docs https://api.flowengineering.com/rest/v1/docs\\n\\nCan I build my own integrations?\\n\\nFor simpler use cases, you can build your own integrations through our API. Our team is happy to help in getting you started.\\n\\nNeed an integration?\\n\\nIntegration roadmap access is available on the Pro and Enterprise plans. Please contact our team to request a new integration. «EMAIL»\\n\\n[Flow logo](https://flowengineering.com/)\\n\\n© Copyright 2025 TRC Space Ltd.\\n\\nAll rights reserved.\\n\\nProviding new-age engineering companies with a requirements tool that is built specifically for their needs and allows them to focus on engineering ground breaking products.\\n\\n[linkedin](https://www.linkedin.com/company/the-engineering-company)\\n[twitter](https://twitter.com/Flow_Engineer)\\n\\nProduct\\n=======\\n\\n[Pricing](https://flowengineering.com/pricing/)\\n\\n[Release notes](https://flow-engineering.productlane.com/changelog)\\n\\n[Security](https://flowengineering.com/security/)\\n\\n[API](https://api.flowengineering.com/rest/v1/docs)\\n\\nCompany\\n=======\\n\\n[About us](https://flowengineering.com/about-us/)\\n\\n[Careers](https://flowengineering.com/careers/)\\n\\n[Press](https://techcrunch.com/2022/12/06/flow-engineering-wants-to-modernize-the-hardware-engineering-design-process/)\\n\\n[Contact](mailto:«EMAIL»)\\n\\nLegal\\n=====\\n\\n[Disclaimer](https://flowengineering.com/disclaimer/)\\n\\n[Privacy and Cookies](https://flowengineering.com/privacy-and-cookie-policy/)\\n\\nResources\\n=========\\n\\n[Iterative Systems Engineering: In Practice\\\\\\n\\\\\\nNew](https://flowengineering.com/blog/iterative-systems-engineering-in-practice-pdf/)\\n\\n[Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n\\n[Iterative Systems Engineering Handbook](https://flowengineering.com/handbook/)\\n\\n[Agile Systems Engineering at SpaceX](https://flowengineering.com/blog/spacex-systems-engineering-five-tips/)\\n\\n[Functional vs Non-Functional Requirements](https://flowengineering.com/blog/different_types-of_requirements_in_a_hardware_engineering_project/)\\n\\n[Blog](https://flowengineering.com/blog/)\\n\\nFlow raises **$23M Series A** from ![Sequoia](https://images.ctfassets.net/c2mtbunjxyfe/DiQDL1MjhDqypgU1DqWEG/1a969dc161a43de7498a3bd5d1686ffa/Frame_48.svg)\\n\\n[Read More](https://flowengineering.com/blog/flow-raises-23m-from-sequoia-to-accelerate-the-future-of-hardware-development/)\\n\\n* [Handbook](https://flowengineering.com/handbook/volume-1/)\\n \\n* [AI](https://flowengineering.com/ai-agent/)\\n \\n* Industries\\n \\n * [Launch Vehicles](https://flowengineering.com/launch-vehicles/)\\n \\n * [Satellites](https://flowengineering.com/satellites/)\\n \\n * [Ground Station](https://flowengineering.com/ground-station/)\\n \\n * [Aviation & Aero](https://flowengineering.com/aviation-and-aero/)\\n \\n * [Fusion](https://flowengineering.com/fusion/)\\n \\n * [Nuclear (SMR)](https://flowengineering.com/nuclear/)\\n \\n * [Quantum](https://flowengineering.com/quantum/)\\n \\n * [eVTOL](https://flowengineering.com/evtol/)\\n \\n * [Automotive](https://flowengineering.com/automotive/)\\n \\n * [Robotics](https://flowengineering.com/robotics/)\\n \\n * [Medical Devices](https://flowengineering.com/medical-devices/)\\n \\n* Product\\n \\n * [Requirements Management](https://flowengineering.com/requirements-management/)\\n \\n * [Architecture](https://flowengineering.com/architecture/)\\n \\n * [Traceability](https://flowengineering.com/traceability/)\\n \\n * [Continuous Verification](https://flowengineering.com/continuous-verification/)\\n \\n * [Live V&V](https://flowengineering.com/live-vv/)\\n \\n * [Test Cases\\\\\\n \\\\\\n new](https://flowengineering.com/test-cases/)\\n \\n * [Regulatory](https://flowengineering.com/regulatory/)\\n \\n * [Integrations](https://flowengineering.com/integrations/)\\n \\n* [Pricing](https://flowengineering.com/pricing/)\\n \\n* Resources\\n \\n * [Blog](https://flowengineering.com/blog/)\\n \\n * [Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n \\n * [Security](https://flowengineering.com/security/)\\n \\n* Company\\n \\n * [About us](https://flowengineering.com/about-us/)\\n \\n * [Careers](https://flowengineering.com/careers/)\\n \\n\\nPricing\\n=======\\n\\nBasic\\n\\n$150\\n\\nper editor/month\\n\\nPro\\n\\n$300\\n\\nper editor/month\\n\\nEnterprise\\n\\n[Contact Us](mailto:«EMAIL»)\\n\\nRequirements Management\\n\\nRequirements\\n\\nUp to 500\\n\\nUp to 5,000\\n\\nUnlimited\\n\\nProjects and Configurations\\n\\n1\\n\\n5\\n\\nUnlimited\\n\\nSecurity and Export Control\\n\\nCloud Security\\n\\nITAR Compliant AWS GovCloud\\n\\nAvailable\\n\\nAPI Access\\n\\nTailored Onboarding and Setup\\n\\nSSO\\n\\nAdditional Integrations on request\\n\\n[Flow logo](https://flowengineering.com/)\\n\\n© Copyright 2025 TRC Space Ltd.\\n\\nAll rights reserved.\\n\\nProviding new-age engineering companies with a requirements tool that is built specifically for their needs and allows them to focus on engineering ground breaking products.\\n\\n[linkedin](https://www.linkedin.com/company/the-engineering-company)\\n[twitter](https://twitter.com/Flow_Engineer)\\n\\nProduct\\n=======\\n\\n[Pricing](https://flowengineering.com/pricing/)\\n\\n[Release notes](https://flow-engineering.productlane.com/changelog)\\n\\n[Security](https://flowengineering.com/security/)\\n\\n[API](https://api.flowengineering.com/rest/v1/docs)\\n\\nCompany\\n=======\\n\\n[About us](https://flowengineering.com/about-us/)\\n\\n[Careers](https://flowengineering.com/careers/)\\n\\n[Press](https://techcrunch.com/2022/12/06/flow-engineering-wants-to-modernize-the-hardware-engineering-design-process/)\\n\\n[Contact](mailto:«EMAIL»)\\n\\nLegal\\n=====\\n\\n[Disclaimer](https://flowengineering.com/disclaimer/)\\n\\n[Privacy and Cookies](https://flowengineering.com/privacy-and-cookie-policy/)\\n\\nResources\\n=========\\n\\n[Iterative Systems Engineering: In Practice\\\\\\n\\\\\\nNew](https://flowengineering.com/blog/iterative-systems-engineering-in-practice-pdf/)\\n\\n[Newsletter](https://flowengineering.com/agile-systems-engineering-briefing/)\\n\\n[Iterative Systems Engineering Handbook](https://flowengineering.com/handbook/)\\n\\n[Agile Systems Engineering at SpaceX](https://flowengineering.com/blog/spacex-systems-engineering-five-tips/)\\n\\n[Functional vs Non-Functional Requirements](https://flowengineering.com/blog/different_types-of_requirements_in_a_hardware_engineering_project/)\\n\\n[Blog](https://flowengineering.com/blog/)\" }, { \"merged_markdown\": \"Infrastructure for Public Good\\n==============================\\n\\nHub Public Benefit Corp., is a Delaware Public Benefit Corporation, committed to providing broad public access to verified knowledge that supports informed decision-making.\\n\\n[About Hub Public Benefit Corp](https://cta-na2.hubspot.com/web-interactives/public/v1/track/click?encryptedPayload=AVxigLKDrcGOnZFt4eUgU7JccsrjodixuNURbPKFsu%2FL3Sd41Tj%2FjT%2F9KLygX81c0A9Q1b5i6hCMIWOc5kLoGXY2COjHzRLMrZlGb2PQjLgRuMkVW%2B4fwiWnznv8IBWmm4kKu9yc4y6%2BskeHdN%2F%2FNtbsf%2F%2FSP1rI86q39A%3D%3D&portalId=7663287&webInteractiveContentId=249984799431&webInteractiveId=578364449509&containerType=EMBEDDED&campaignId=14ee777f-ccbf-491e-8b5d-b10b3338320f&pageUrl=https%3A%2F%2Fhub.inc%2F&pageTitle=Hub+-+Public+Benefit+Corporation&referrer=&userAgent=Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F141.0.0.0+Safari%2F537.36&hutk=75386abdd4419d41b47b956559d642d4&hssc=209250317.1.1762764655843&hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&pageId=228263428829&analyticsPageId=228263428829&hsfp=3182208643&canonicalUrl=https%3A%2F%2Fhub.inc&contentType=standard-page&__hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&__hssc=209250317.1.1762764655843&__hsfp=3182208643)\\n\\nPHapp®\\n======\\n\\nAn initiative by Hub Public Benefit Corp to help the public stay informed and prepared with this free app and website that delivers verified public health and safety information in 50+ languages.\\n\\n[Learn about PHapp](https://cta-na2.hubspot.com/web-interactives/public/v1/track/click?encryptedPayload=AVxigLIeSlALqnZbwJA9nj5vc7z8JCcykCvxXsCa1JAQ5EFvNDNHwWKPgq3XELzVM%2B2AbzR1qImCg%2B%2FxyjtZLyv06hpxuJJQm0o1%2F9UEim80avWFwvTA4FlCZ0gvs0qIVZ7yFf8oYB%2BUy06BMqjmRKL3c%2BCeJO1m8bs%3D&portalId=7663287&webInteractiveContentId=249984799431&webInteractiveId=578364449509&containerType=EMBEDDED&campaignId=14ee777f-ccbf-491e-8b5d-b10b3338320f&pageUrl=https%3A%2F%2Fhub.inc%2F&pageTitle=Hub+-+Public+Benefit+Corporation&referrer=&userAgent=Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F141.0.0.0+Safari%2F537.36&hutk=75386abdd4419d41b47b956559d642d4&hssc=209250317.1.1762764655843&hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&pageId=228263428829&analyticsPageId=228263428829&hsfp=3182208643&canonicalUrl=https%3A%2F%2Fhub.inc&contentType=standard-page&__hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&__hssc=209250317.1.1762764655843&__hsfp=3182208643)\\n\\n![App_01_9.16](https://hub.inc/hs-fs/hubfs/Assets/App_01_9.16.png?width=2484&height=4416&name=App_01_9.16.png)\\n\\nWehealth®\\n=========\\n\\nAn initiative by Hub Public Benefit Corp. to improve community resilience worldwide by bringing together inter-disciplinary experts and community stakeholders to co-develop and continuously improve outcomes.\\n\\n[Learn about Wehealth](https://cta-na2.hubspot.com/web-interactives/public/v1/track/click?encryptedPayload=AVxigLLmtRVPqVlx85rxucWatsgEEOSM6AszEW5i4k%2Bo9V80zibaC83mDVMgQ78JWUwFI16wh0uGlSvoBX0LihTJzSi%2Ba3LTaPinZBbdfVZJI42phQxP2WQoteFpRp%2FqKQxM1nTuTl50ojS4WuEjrI4meRKc47nWEbtqV4RmLxIW&portalId=7663287&webInteractiveContentId=249984799431&webInteractiveId=578364449509&containerType=EMBEDDED&campaignId=14ee777f-ccbf-491e-8b5d-b10b3338320f&pageUrl=https%3A%2F%2Fhub.inc%2F&pageTitle=Hub+-+Public+Benefit+Corporation&referrer=&userAgent=Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F141.0.0.0+Safari%2F537.36&hutk=75386abdd4419d41b47b956559d642d4&hssc=209250317.1.1762764655843&hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&pageId=228263428829&analyticsPageId=228263428829&hsfp=3182208643&canonicalUrl=https%3A%2F%2Fhub.inc&contentType=standard-page&__hstc=209250317.75386abdd4419d41b47b956559d642d4.1762764655843.1762764655843.1762764655843.1&__hssc=209250317.1.1762764655843&__hsfp=3182208643)\\n\\n![Feature_Portal_01](https://hub.inc/hs-fs/hubfs/Assets/Feature_Portal_01.png?width=1116&height=1899&name=Feature_Portal_01.png)\" } ]", "type": "array" } ] }, "options": {} }, "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 352, 0 ], "id": "333e0707-6cc7-42de-a963-b78d51b501f1", "name": "ICP" }, { "parameters": { "promptType": "define", "text": "=URL: {{ $('Filter').item.json.Website }}", "hasOutputParser": true, "options": { "systemMessage": "Always use your web search tool.\n\nYou are an ICP scoring system for identifying high-ticket B2B SaaS companies ($25k+ deals). Analyze the provided URL by searching only the homepage and pricing page to determine fit. Target: B2B SaaS for businesses. AVOID: B2C SaaS, Advertising/Marketing/Sales Software, E-Commerce targeting companies (ecommerce software/marketplaces), Managed IT Services/Staffing, Higher Education SaaS, Events/Planners/Conferences SaaS, Web3/Blockchain/Cryptocurrency software, Government SaaS, Salesforce partners, Medical Billing/Clinic Scheduling software. If obviously not software/SaaS or has clear low monthly pricing ($250-500/month), score 3 or below—only after confirmation to avoid false lows.\n\nResearch Method:\n1. Search only the homepage and pricing page of the provided URL.\n2. Verify alignment: Check for B2B SaaS indicators (e.g., \"enterprise software\"). Negative if matches avoid list (e.g., \"marketing tool\", \"ecommerce platform\").\n3. Check audience: Infer from homepage/pricing for business focus. Negative if avoid categories (e.g., \"for consumers\", \"government use\").\n4. Pricing clues: Look for low plans ($250-500/month)—negative; favor custom/enterprise.\n5. If search blocked or insufficient data, output empty string. Otherwise, assign at least 1.\n\nClassification: Rate 1-10:\n- 1–3: Not a fit (confirmed avoid category, not software/SaaS, or explicit low pricing).\n- 4–6: Moderate (some B2B but unclear/overlap).\n- 7–8: Good (strong B2B SaaS, no avoids, higher pricing hints).\n- 9–10: Perfect (clear high-ticket B2B SaaS, no negatives).\n\nOutput format: Return ONLY a JSON object: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text. If no viable score (e.g., blocked search), return empty string \"\"." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 880, 368 ], "id": "57b96542-136e-4977-8254-7e83c7d6b07c", "name": "AI Agent" }, { "parameters": { "toolDescription": "Web search", "url": "https://www.googleapis.com/customsearch/v1", "sendQuery": true, "queryParameters": { "parameters": [ { "name": "key", "value": "«GOOGLE_API_KEY»" }, { "name": "cx", "value": "82839611380ce4bfc" }, { "name": "q", "value": "={{ /*n8n-auto-generated-fromAI-override*/ $fromAI('parameters2_Value', ``, 'string') }}" }, { "name": "num", "value": "5" } ] }, "options": { "timeout": 10000 } }, "type": "n8n-nodes-base.httpRequestTool", "typeVersion": 4.3, "position": [ 896, 544 ], "id": "ee781931-f161-492a-86ba-a83f6ca941cc", "name": "Web Search" }, { "parameters": { "jsonSchemaExample": "{\n \"ICP Score\": \"7\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1008, 544 ], "id": "970c09b7-d527-4199-93f4-b6b4b48578be", "name": "Structured Output Parser1" }, { "parameters": { "jsonSchemaExample": "{\n \"ICP Score\": \"7\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1568, 160 ], "id": "bb0da618-8a6a-4b8f-9177-a7b6de9f443d", "name": "Structured Output Parser4" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-mini", "mode": "list", "cachedResultName": "gpt-5-mini" }, "options": { "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1424, 160 ], "id": "130666e3-0045-4450-b282-26a2dbef6fd5", "name": "OpenAI Chat Model3", "credentials": { "openAiApi": { "id": "UggzGrwKVc28ZAlJ", "name": "Local" } } }, { "parameters": { "promptType": "define", "text": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}", "hasOutputParser": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1424, 0 ], "id": "084feef3-2ece-44f1-ac77-63c67be81d52", "name": "Format Names" } ], "connections": { "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Case Study", "type": "main", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "ICP + Service", "type": "ai_outputParser", "index": 0 } ] ] }, "Structured Output Parser2": { "ai_outputParser": [ [ { "node": "Case Study ICP", "type": "ai_outputParser", "index": 0 } ] ] }, "Case Study ICP": { "main": [ [ { "node": "ICP + Service", "type": "main", "index": 0 } ] ] }, "ICP + Service": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "Structured Output Parser3": { "ai_outputParser": [ [ { "node": "ICP Scoring", "type": "ai_outputParser", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Case Study ICP", "type": "ai_languageModel", "index": 0 } ] ] }, "OpenAI Chat Model1": { "ai_languageModel": [ [ { "node": "ICP + Service", "type": "ai_languageModel", "index": 0 } ] ] }, "OpenAI Chat Model2": { "ai_languageModel": [ [ { "node": "ICP Scoring", "type": "ai_languageModel", "index": 0 } ] ] }, "Case Study": { "main": [ [ { "node": "ICP", "type": "main", "index": 0 } ] ] }, "ICP": { "main": [ [ { "node": "Case Study ICP", "type": "main", "index": 0 } ] ] }, "Web Search": { "ai_tool": [ [ { "node": "AI Agent", "type": "ai_tool", "index": 0 } ] ] }, "Structured Output Parser1": { "ai_outputParser": [ [ { "node": "AI Agent", "type": "ai_outputParser", "index": 0 } ] ] }, "Structured Output Parser4": { "ai_outputParser": [ [ { "node": "Format Names", "type": "ai_outputParser", "index": 0 } ] ] }, "OpenAI Chat Model3": { "ai_languageModel": [ [ { "node": "Format Names", "type": "ai_languageModel", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "92f056b8-ad40-4fd3-ba31-a3311d5a811f", "activeVersionId": null, "versionCounter": 35, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-08T05:55:57.952Z", "createdAt": "2025-11-08T05:55:57.952Z", "role": "workflow:owner", "workflowId": "1wTlVvSGibxy80WY", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-12T19:11:03.000Z", "createdAt": "2025-11-11T03:33:22.317Z", "id": "xeXJBJJOqM79ziC0", "name": "客戶A - Crawl4AI", "description": null, "active": false, "isArchived": true, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 496, 112 ], "id": "be023540-3939-45e8-a1ac-a3e374f607c7", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n Firecrawl Input", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 656, 112 ], "id": "0f115cd8-944e-4e98-916d-84ca3805d36b", "name": "Get row(s) in sheet" }, { "parameters": { "batchSize": 5, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 976, 112 ], "id": "0591fc95-fb2b-4547-b51a-6f9f32870db0", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1456, 176 ], "id": "721797b1-8134-400f-a11e-1bb3da00c648", "name": "Get Crawl Dataset", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1616, 176 ], "id": "3547ac0e-2ca9-478c-83c7-9d652c5c0bdc", "name": "If" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ 816, 112 ], "id": "cf282dce-30f6-4782-a041-d70f16fbe2a4", "name": "Filter" }, { "parameters": { "content": "## Step 2: Extract data from firecrawl output\n ", "height": 448, "width": 1632 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 640 ], "typeVersion": 1, "id": "d26ba9e8-b0e5-40f7-bbe1-630e53be3246", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 448, "width": 464, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ 2000, 640 ], "typeVersion": 1, "id": "bb00de20-8337-4f4c-847f-e2ff687f6178", "name": "Sticky Note2" }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "const merged = $input.item.json.merged_markdown; // 從前輸出取 merged_markdown\nconst front = merged.slice(0, 1000);\nconst back = merged.slice(-1000);\nconst shortened = front + back;\nreturn { json: { shortened_markdown: shortened } }; // 輸出 shortened_markdown" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 976, 448 ], "id": "f0496daa-905b-44ee-894c-4e48815d559f", "name": "Shorten Markdown", "alwaysOutputData": true }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1936, 192 ], "id": "d6b90c59-c83b-4eaf-adfb-17297e90fcd7", "name": "Get Crawl Dataset1", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2096, 192 ], "id": "b78782e1-7547-4d3d-b4df-cc82767d60f7", "name": "If1" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 448 ], "id": "ae93a34f-9abf-40d0-88cb-d423dc0b15bd", "name": "Get Crawl Dataset2", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 20 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1776, 192 ], "id": "42437e0f-577d-40f3-a588-492746feb689", "name": "20 sec", "webhookId": "56a3d49c-e43e-46b6-93c8-c7267edbb7e7" }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1296, 176 ], "id": "5487efe2-5b41-4a4e-ab68-07def708287f", "name": "10 sec", "webhookId": "a803222f-c767-4d17-ba7a-13ea9ac181b2" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 2032, 448 ], "id": "2ceb7c6e-5214-42be-8dd5-291461b79b75", "name": "Get Crawl Dataset3", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2192, 448 ], "id": "421f8323-f276-4f10-8a0c-ebcf6a694f8c", "name": "If2" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 752 ], "id": "c89cadcf-168a-4d35-94ef-b0b4efe43a4b", "name": "Get Crawl Dataset4", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1872, 448 ], "id": "8f18d6cc-b079-47ac-ac61-75b5e0b8eb26", "name": "10 sec1", "webhookId": "2c1cbe2e-c5de-4f49-9921-a01942d9ac45" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ☆ Qualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json[\"First Name\"] }}", "last name": "={{ $('Get row(s) in sheet').item.json[\"Last Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json[\"Person Linkedin Url\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 912 ], "id": "54ab3b12-bb4a-4efd-a102-ff2bd6814188", "name": "Qualified" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ⨯ Unqualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 736 ], "id": "5d110fcf-2cd8-45d3-adbb-cca051e5b1bb", "name": "Unqualified" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "cf6d927d-baf8-412b-911a-77d4cfbaf98e", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "1", "operator": { "type": "string", "operation": "equals" } }, { "id": "a71d0bb7-5845-46df-b5d8-d2fab0477bcb", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "2", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "7868fe66-3516-4d01-90e1-32aaafc0f475", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "3", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "24a7144e-0116-4040-b5b1-7b3fc6f46568", "leftValue": "={{ $json.output.ICP }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "ea2c8572-edab-4bbc-9100-29ba53c21b61", "leftValue": "={{ $json.output[\"Service Benefit\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "014459a4-db94-4efa-a4f3-683d8d1470f9", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1840, 752 ], "id": "cfa7925b-e112-468b-8a8a-b361a98f5b13", "name": "If Score < 3" }, { "parameters": {}, "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 2240, 752 ], "id": "0d64db45-eca4-45cc-bf1c-c673f8bb3dd0", "name": "Merge" }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $('Filter').item.json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"app/.*\",\n \"demo/.*\",\n \"news/.*\",\n \"events/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"home/.*\",\n \"pricing/.*\",\n \"about/.*\",\n \"products/.*\",\n \"services/.*\",\n \"company/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"benefits/.*\",\n \"integrations/.*\",\n \"target-audience/.*\",\n \"platform/.*\",\n \"why/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1712, 448 ], "id": "86b5204a-932e-4cd2-862b-683516392cca", "name": "Crawl ICP", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"pricing/.*\",\n \"products/.*\",\n \"services/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\",\n \"home/.*\",\n \"about/.*\",\n \"privacy/.*\",\n \"cookies/.*\",\n \"blog-preview/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customers/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1136, 176 ], "id": "6430fb14-94f4-4871-9f1d-304781143e14", "name": "Crawl Case Study", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 448 ], "id": "e82835c4-bfca-4db5-9d80-f062b5827dfe", "name": "30 sec", "webhookId": "2552e1e2-8883-4480-a9ea-1c3266e71c81" }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.5-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('Case Study Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for estimated Ideal Customer Profile (ICP) from crawled web page markdown focused on case studies. Your task is to process the provided crawled page markdown and extract an estimated ICP based on client companies mentioned in case studies, outputting it in JSON format inside an \"output\" key.\n\n\"Estimated ICP\": Extract a short phrase describing the ideal customer profile (ICP) as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\"). Base it primarily on client companies found in case studies, inferring their industries or types only if unclear from markdown.\n- Scope: Search ONLY in the provided markdown initially. Prefer sections like: \"Case Studies\", \"Success Stories\", \"Clients\", \"Testimonials\", \"Projects\", \"Portfolio\", \"Our Work\", \"Results\", \"Reviews\".\n- What counts as a case study client: Named companies explicitly presented as clients in case studies, projects, success stories, or testimonials, with descriptions of work done for them. Accept names in headings, titles, attributions, or body text where context indicates client relationship. Always exclude the host company's own name (provided in user input) to avoid self-referencing.\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\").\n - Format: Company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple industries, infer the most common or dominant one based on frequency of case studies, prioritizing precision over breadth.\n - Analyze descriptions: From markdown, identify industries or company types of the clients (e.g., if a case study describes \"helping an e-commerce retailer optimize logistics\", infer \"e-commerce businesses\").\n - If multiple case studies cluster in one industry (e.g., several in finance), set ICP to that (e.g., \"investment banks and asset managers\").\n - Preserve logical grouping; avoid broad terms like \"all businesses\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include the host company's own name, products, partners, or non-client entities. If a company name matches the host company, ignore it entirely.\n - If industry unclear from markdown: As a last resort only (if completely no info after exhaustive analysis), use tools to search the web for the client company's industry (e.g., web_search with query like \"what industry is [company name] in? site:wikipedia.org\" or reliable sources). Limit to 1-2 searches per unclear client, and only for confirmed client names excluding the host company. Integrate results to refine ICP without guessing.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) At least one (preferably multiple) explicit case study clients with context indicating they are customers (e.g., \"We helped [company] achieve X\").\n B) The inferred ICP is based on company types/industries/roles, not products, awards, or vague mentions.\n C) The phrase is contiguous and directly mappable to client descriptions; no guessing.\n D) If searching externally, results must confirm industry without contradiction, and searches must exclude the host company name.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by” – ignore these entirely.\n - Logo walls or lists without case study details or clear client labeling.\n - Ambiguous mentions where companies could be partners/vendors, not clients.\n - Fewer than 2 case studies, or no clear industry clustering.\n - External search yields no reliable industry info or conflicts with markdown.\n- If NOT FOUND, NOT CERTAIN, or no viable ICP: Return an empty string \"\". Do NOT write \"N/A\", \"red\", \"None\", or any explanation. Do not output search suggestions if using tools – integrate directly.\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Estimated ICP\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty.", "codeExecution": false, "maxOutputTokens": 1000, "candidateCount": 1, "temperature": 0.2, "topP": 1, "topK": 40 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1136, 448 ], "id": "3e1b5f87-a883-4dd5-baf1-7d79abf49f17", "name": "Case Study ICP", "alwaysOutputData": false, "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash", "mode": "list", "cachedResultName": "models/gemini-2.5-flash" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $json.merged_markdown }}\nEstimated ICP: {{ $('Case Study ICP').item.json.content.parts[0].text }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.\n\n\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").\n- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow.\n - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth.\n - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit.\n - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include products, partners, or non-customer entities.\n - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort):\n A) Explicit mentions of target customers in relevant sections.\n B) The inferred ICP is based on company types/industries/roles, not vague or broad statements.\n C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence.\n D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\n\n\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").\n- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points).\n - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text.\n - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation.\n - If multiple products/services, select the primary or most emphasized one.\n - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation.\n - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains).\n - Do NOT guess; if unclear, treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) Explicit description of primary product/service and its benefit.\n B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision.\n C) Directly from text; no invention, and it must support logical flow in the target sentence.\n D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).\n\nOverall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.\n\n- Disqualifiers (any one triggers NOT FOUND for the field):\n - Vague or no explicit mentions in preferred sections.\n - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages.\n - Ambiguous or conflicting info.\n- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.\n\nFinal Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 3000, "candidateCount": 1 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 976, 752 ], "id": "eb980838-f0f4-4fdf-9526-7ea3ba195336", "name": "ICP + Service", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('ICP Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are an ICP scoring system for identifying B2B SaaS companies that align with high-ticket deals ($25k+). Analyze the provided crawled markdown from the company's website to determine fit based on their services, target audience, and any visible pricing indicators. Objective: Identify companies offering B2B SaaS solutions targeted at businesses, excluding B2C-focused SaaS, advertising/marketing/sales software, e-commerce software or marketplaces, managed IT services/staffing, higher education SaaS, events/planners/conferences SaaS, Web3/blockchain/cryptocurrency software, government SaaS, Salesforce partners, medical billing services/software, or clinic/medical scheduling software. Prioritize indications of higher-ticket pricing (avoid low monthly plans like $250-500/month if mentioned; favor custom/enterprise pricing).\n\nResearch Method:\n\n1. Review the crawled markdown content from the company's website, focusing on sections like homepage, about us, products/services, solutions, case studies, testimonials, pricing, and blog.\n\n2. Verify service alignment: Scan for keywords indicating B2B SaaS (e.g., \"enterprise software\", \"business solutions\", \"B2B platform\"). Confirm services are for businesses, not the avoided categories. For example:\n - Positive: Mentions \"B2B tools for companies\", \"enterprise automation\".\n - Negative: Keywords like \"B2C app\", \"marketing software\", \"sales CRM\", \"ecommerce platform\", \"IT managed services\", \"staffing solutions\", \"edtech\", \"event management\", \"blockchain software\", \"govtech\", \"Salesforce partner\", \"medical billing\", \"clinic scheduling\".\n\n3. Infer target audience: Analyze case studies, client testimonials, or \"who we serve\" sections for business/enterprise focus. For example:\n - Positive: Clients as \"businesses in tech/finance\", \"enterprise users\".\n - Negative: Mentions \"consumers\", \"e-commerce stores\", \"universities\", \"event planners\", \"government agencies\", \"healthcare clinics\".\n\n4. Check for pricing clues: Look for mentions like \"starting at $X/month\" or \"enterprise pricing\". For example:\n - Positive: \"Custom pricing\", \"contact for quote\", \"annual contracts for enterprises\" (suggesting $25k+).\n - Negative: \"Starting at $250-500/month\", \"low-cost plans\" (indicating low-ticket).\n\n5. Overall fit assessment: Weigh positive vs. negative indicators. If ambiguous, lean toward moderate scores rather than low.\n\nClassification: Rate on a scale from 1 to 10:\n- 1–3: Not a fit (use only if unavoidable or definitively matches an avoided category, e.g., clearly in avoid list like B2C SaaS or low-ticket with explicit $250-500/month pricing; reserve for confirmed mismatches).\n- 4–6: Moderate fit (e.g., some B2B elements but potential overlap with avoided areas, unclear audience, or mixed pricing signals).\n- 7–8: Good fit (e.g., strong B2B SaaS focus, no avoids, suggests higher pricing).\n- 9–10: Perfect fit (e.g., high-ticket B2B SaaS with clear business targeting, no negatives).\n\nDeliverable: Provide ONLY the rating for the company on a scale from 1 to 10, with no additional explanation or details.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty. If no viable score (e.g., insufficient data), return \"0\".", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1264, 752 ], "id": "68207a97-2a7c-461f-81b9-f3b07db2580d", "name": "ICP Scoring", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1424, 448 ], "id": "c30e3042-ba80-4df0-883e-9de6cda57999", "name": "Format Names", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 448 ], "id": "a250b021-4837-42e0-9f21-5125677b8dcc", "name": "Case Study Merge", "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 752 ], "id": "e6752ed5-c6e4-494c-932e-cecbfef72466", "name": "ICP Merge", "alwaysOutputData": true }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 752 ], "id": "4d033130-0437-48d7-9ad6-39d61d4a6762", "name": "30 sec1", "webhookId": "75c20d6d-76fa-4cb5-8c67-c215c2b37e2a" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "responseFormat": "json_object", "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1552, 912 ], "id": "0a67a5ca-b0dd-4359-8867-3ed2c9d38dc1", "name": "OpenAI Chat Model" }, { "parameters": { "jsonSchemaExample": "{\n\t\"ICP\": \"iterative hardware teams\",\n \"Service Benefit\": \"accelerate talent sourcing\",\n \"Company Name\": \"Cyberleaf\",\n \"Job Title\": \"Regional Director Sales\",\n \"ICP Score\": \"8\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1696, 912 ], "id": "805471a8-0763-4489-bfc2-2b62c8da57dd", "name": "Structured Output Parser" }, { "parameters": { "promptType": "define", "text": "=Process these input strings and extract the fields ICP, Service Benefit, Company Name, Job Title, ICP Score into a clean single JSON object:\n{{ $json.content.parts[0].text }}\n{{ $('ICP + Service').item.json.content.parts[0].text}}\n{{ $('Format Names').item.json.content.parts[0].text }}", "hasOutputParser": true, "options": { "systemMessage": "You are a JSON data cleaner and organizer for extracting and structuring specific fields from input strings. \n\nYour task is to process the provided input strings, which may contain JSON objects or arrays, and extract values for the fields: ICP, Service Benefit, Company Name, Job Title, ICP Score. \n\nObjective: \nTransform the inputs into a clean single object that merges all extracted fields from the inputs, filling missing fields with empty strings. \n\nResearch Method: \n1. Parse each input string as JSON, handling potential formatting issues like escaped newlines. \n2. For each parsed structure, navigate to the innermost object (e.g., under \"output\") and extract the specified fields if present. \n3. If an input is an array, process each element separately. \n4. Merge all extracted data into a single object, overwriting duplicates if any (use the last value encountered). \n5. If parsing fails or no fields match, leave the corresponding field as an empty string in the output object. \n\nClassification: \nEnsure fields are extracted exactly as strings; do not infer or add data. \n\nDeliverable: \nProvide ONLY the extracted data as a single object, with no additional explanation or details. \n\nOutput format: \nReturn ONLY a JSON object in this exact structure: {\"ICP\": \"value or empty string\", \"Service Benefit\": \"value or empty string\", \"Company Name\": \"value or empty string\", \"Job Title\": \"value or empty string\", \"ICP Score\": \"value or empty string\"}. No additional text, commentary, or formatting. Always output a single object, even if empty or with empty strings." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1552, 752 ], "id": "3c5b93e9-8c29-45ab-aab4-166d6ad3d667", "name": "Clean" }, { "parameters": { "content": "## Step 1: Crawl websites using firecrawl\n", "height": 608, "width": 2096, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 32 ], "typeVersion": 1, "id": "4e8c453a-c367-4aa2-9011-a23d4a7c2ca6", "name": "Sticky Note" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $json.merged_markdown }}\nEstimated ICP: {{ $('Case Study ICP').item.json.content.parts[0].text }}", "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.|\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".- Strict rules: - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow. - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused). - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth. - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit. - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus. - Do NOT include products, partners, or non-customer entities. - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort): A) Explicit mentions of target customers in relevant sections. B) The inferred ICP is based on company types/industries/roles, not vague or broad statements. C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence. D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".- Strict rules: - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points). - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text. - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation. - If multiple products/services, select the primary or most emphasized one. - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation. - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains). - Do NOT guess; if unclear, treat as NOT FOUND.- Certainty gate (must ALL be true, otherwise treat as NOT FOUND): A) Explicit description of primary product/service and its benefit. B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision. C) Directly from text; no invention, and it must support logical flow in the target sentence. D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).Overall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.- Disqualifiers (any one triggers NOT FOUND for the field): - Vague or no explicit mentions in preferred sections. - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages. - Ambiguous or conflicting info.- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.Final Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.Output format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 592, 1600 ], "id": "2836bd1a-5988-4119-94ba-998fb64de2c6", "name": "AI Agent" }, { "parameters": { "options": {} }, "type": "@n8n/n8n-nodes-langchain.lmChatGoogleGemini", "typeVersion": 1, "position": [ 592, 1792 ], "id": "09311fd2-e53f-4eab-ae08-f248044de85f", "name": "Google Gemini Chat Model", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } } ], "connections": { "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Get row(s) in sheet", "type": "main", "index": 0 } ] ] }, "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "Crawl Case Study", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset": { "main": [ [ { "node": "If", "type": "main", "index": 0 } ] ] }, "If": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "20 sec", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Shorten Markdown": { "main": [ [ { "node": "Case Study ICP", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset1": { "main": [ [ { "node": "If1", "type": "main", "index": 0 } ] ] }, "If1": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset2": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ] ] }, "20 sec": { "main": [ [ { "node": "Get Crawl Dataset1", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get Crawl Dataset", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset3": { "main": [ [ { "node": "If2", "type": "main", "index": 0 } ] ] }, "If2": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec1", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset4": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ] ] }, "10 sec1": { "main": [ [ { "node": "Get Crawl Dataset3", "type": "main", "index": 0 } ] ] }, "Qualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Unqualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "If Score < 3": { "main": [ [ { "node": "Unqualified", "type": "main", "index": 0 } ], [ { "node": "Qualified", "type": "main", "index": 0 } ] ] }, "Merge": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Crawl ICP": { "main": [ [ { "node": "10 sec1", "type": "main", "index": 0 } ] ] }, "Crawl Case Study": { "main": [ [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "30 sec": { "main": [ [ { "node": "Get Crawl Dataset2", "type": "main", "index": 0 } ] ] }, "Case Study ICP": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] }, "ICP + Service": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "Clean", "type": "main", "index": 0 } ] ] }, "Format Names": { "main": [ [ { "node": "Crawl ICP", "type": "main", "index": 0 } ] ] }, "Case Study Merge": { "main": [ [ { "node": "Shorten Markdown", "type": "main", "index": 0 } ] ] }, "ICP Merge": { "main": [ [ { "node": "ICP + Service", "type": "main", "index": 0 } ] ] }, "30 sec1": { "main": [ [ { "node": "Get Crawl Dataset4", "type": "main", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Clean", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "Clean", "type": "ai_outputParser", "index": 0 } ] ] }, "Clean": { "main": [ [ { "node": "If Score < 3", "type": "main", "index": 0 } ] ] }, "Google Gemini Chat Model": { "ai_languageModel": [ [ { "node": "AI Agent", "type": "ai_languageModel", "index": 0 } ] ] }, "AI Agent": { "main": [ [] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "5cffd38d-45bc-41c7-ac42-5725f0e6ce61", "activeVersionId": null, "versionCounter": 69, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-11T03:33:22.322Z", "createdAt": "2025-11-11T03:33:22.322Z", "role": "workflow:owner", "workflowId": "xeXJBJJOqM79ziC0", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-20T20:40:02.000Z", "createdAt": "2025-11-12T19:09:08.132Z", "id": "RpVqIfqqkTXXGcB3", "name": "客戶A - Firecrawl", "description": null, "active": false, "isArchived": true, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 496, 112 ], "id": "97f47484-823a-4531-83dc-a131cb1bd83d", "name": "When clicking ‘Execute workflow’" }, { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n Firecrawl Input", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 656, 112 ], "id": "008a4c6f-ff8d-4eed-a93d-64bbc9d3c0fd", "name": "Get row(s) in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "batchSize": 5, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 976, 112 ], "id": "b0b98425-b5ee-4aba-a6fe-a9127c7c29a1", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1456, 176 ], "id": "5a7e781e-cba6-4bf5-b8af-4d8756aeb292", "name": "Get Crawl Dataset", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1616, 176 ], "id": "89873866-11ab-4de6-8141-1942d36a453f", "name": "If" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ 816, 112 ], "id": "4548f804-50dc-4850-b13c-1c164ae3008c", "name": "Filter" }, { "parameters": { "content": "## Step 2: Extract data from firecrawl output\n ", "height": 448, "width": 1632 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 640 ], "typeVersion": 1, "id": "aa73bbd8-4f96-4ec0-bfa3-407ad9f44e01", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 448, "width": 464, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ 2000, 640 ], "typeVersion": 1, "id": "bc20f952-fedd-465f-853c-8769397bfdde", "name": "Sticky Note2" }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "const merged = $input.item.json.merged_markdown; // 從前輸出取 merged_markdown\nconst front = merged.slice(0, 1000);\nconst back = merged.slice(-1000);\nconst shortened = front + back;\nreturn { json: { shortened_markdown: shortened } }; // 輸出 shortened_markdown" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 976, 448 ], "id": "8cc6f397-675a-4b63-987f-2298bfcf11fe", "name": "Shorten Markdown", "alwaysOutputData": true }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1936, 192 ], "id": "35350752-5b48-4c90-ade1-97553b96003a", "name": "Get Crawl Dataset1", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2096, 192 ], "id": "50133514-6f89-452f-8f03-231fb1d3d673", "name": "If1" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 448 ], "id": "4485e955-de8e-4850-8c49-bb1125d43805", "name": "Get Crawl Dataset2", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 20 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1776, 192 ], "id": "0dbf76d6-c779-4a2c-b7a7-aff5f4d7ca95", "name": "20 sec", "webhookId": "4daada68-5b51-42b4-8307-3a7a3b160665" }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1296, 176 ], "id": "d5673b94-c43d-4f4e-b74c-72afc56a92dd", "name": "10 sec", "webhookId": "2765ddc6-5b45-496a-8cce-3c3be41ec5e8" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 2032, 448 ], "id": "dcfdfe21-8dd2-4863-8a88-a992b472aafe", "name": "Get Crawl Dataset3", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2192, 448 ], "id": "287a566b-a91a-4564-bc0d-f751178115bb", "name": "If2" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 752 ], "id": "3624e96b-678e-41b7-8182-183bc6663013", "name": "Get Crawl Dataset4", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1872, 448 ], "id": "09f107cc-2ff0-4f39-8b7e-9c337f27d51e", "name": "10 sec1", "webhookId": "2765ddc6-5b45-496a-8cce-3c3be41ec5e8" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ☆ Qualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json[\"First Name\"] }}", "last name": "={{ $('Get row(s) in sheet').item.json[\"Last Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json[\"Person Linkedin Url\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 912 ], "id": "a92ff9be-6cf8-41fc-bcdd-78863ac19664", "name": "Qualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ⨯ Unqualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 736 ], "id": "55222a23-f4b1-4545-ae5e-f67e0481ff45", "name": "Unqualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "cf6d927d-baf8-412b-911a-77d4cfbaf98e", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "1", "operator": { "type": "string", "operation": "equals" } }, { "id": "a71d0bb7-5845-46df-b5d8-d2fab0477bcb", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "2", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "7868fe66-3516-4d01-90e1-32aaafc0f475", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "3", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "24a7144e-0116-4040-b5b1-7b3fc6f46568", "leftValue": "={{ $json.output.ICP }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "ea2c8572-edab-4bbc-9100-29ba53c21b61", "leftValue": "={{ $json.output[\"Service Benefit\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "014459a4-db94-4efa-a4f3-683d8d1470f9", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1840, 752 ], "id": "8b01b93a-a407-4e84-a0c8-e609f84aec4a", "name": "If Score < 3" }, { "parameters": {}, "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 2240, 752 ], "id": "511b8803-bad1-47a4-8f95-1edad9067e29", "name": "Merge" }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $('Filter').item.json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"app/.*\",\n \"demo/.*\",\n \"news/.*\",\n \"events/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"home/.*\",\n \"pricing/.*\",\n \"about/.*\",\n \"products/.*\",\n \"services/.*\",\n \"company/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"benefits/.*\",\n \"integrations/.*\",\n \"target-audience/.*\",\n \"platform/.*\",\n \"why/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1712, 448 ], "id": "2f82ee25-dac5-4d9c-989d-dd95cc718bc6", "name": "Crawl ICP", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"pricing/.*\",\n \"products/.*\",\n \"services/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\",\n \"home/.*\",\n \"about/.*\",\n \"privacy/.*\",\n \"cookies/.*\",\n \"blog-preview/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customers/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1136, 176 ], "id": "e22cf7c6-c6c7-46a9-befd-8ef35f8aa4d9", "name": "Crawl Case Study", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 448 ], "id": "abf3decf-4d35-4e79-8a97-f8a84256e73e", "name": "30 sec", "webhookId": "e3f7a2c6-ad79-4127-b998-d8c3097cdfb1" }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.5-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('Case Study Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for estimated Ideal Customer Profile (ICP) from crawled web page markdown focused on case studies. Your task is to process the provided crawled page markdown and extract an estimated ICP based on client companies mentioned in case studies, outputting it in JSON format inside an \"output\" key.\n\n\"Estimated ICP\": Extract a short phrase describing the ideal customer profile (ICP) as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\"). Base it primarily on client companies found in case studies, inferring their industries or types only if unclear from markdown.\n- Scope: Search ONLY in the provided markdown initially. Prefer sections like: \"Case Studies\", \"Success Stories\", \"Clients\", \"Testimonials\", \"Projects\", \"Portfolio\", \"Our Work\", \"Results\", \"Reviews\".\n- What counts as a case study client: Named companies explicitly presented as clients in case studies, projects, success stories, or testimonials, with descriptions of work done for them. Accept names in headings, titles, attributions, or body text where context indicates client relationship. Always exclude the host company's own name (provided in user input) to avoid self-referencing.\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\").\n - Format: Company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple industries, infer the most common or dominant one based on frequency of case studies, prioritizing precision over breadth.\n - Analyze descriptions: From markdown, identify industries or company types of the clients (e.g., if a case study describes \"helping an e-commerce retailer optimize logistics\", infer \"e-commerce businesses\").\n - If multiple case studies cluster in one industry (e.g., several in finance), set ICP to that (e.g., \"investment banks and asset managers\").\n - Preserve logical grouping; avoid broad terms like \"all businesses\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include the host company's own name, products, partners, or non-client entities. If a company name matches the host company, ignore it entirely.\n - If industry unclear from markdown: As a last resort only (if completely no info after exhaustive analysis), use tools to search the web for the client company's industry (e.g., web_search with query like \"what industry is [company name] in? site:wikipedia.org\" or reliable sources). Limit to 1-2 searches per unclear client, and only for confirmed client names excluding the host company. Integrate results to refine ICP without guessing.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) At least one (preferably multiple) explicit case study clients with context indicating they are customers (e.g., \"We helped [company] achieve X\").\n B) The inferred ICP is based on company types/industries/roles, not products, awards, or vague mentions.\n C) The phrase is contiguous and directly mappable to client descriptions; no guessing.\n D) If searching externally, results must confirm industry without contradiction, and searches must exclude the host company name.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by” – ignore these entirely.\n - Logo walls or lists without case study details or clear client labeling.\n - Ambiguous mentions where companies could be partners/vendors, not clients.\n - Fewer than 2 case studies, or no clear industry clustering.\n - External search yields no reliable industry info or conflicts with markdown.\n- If NOT FOUND, NOT CERTAIN, or no viable ICP: Return an empty string \"\". Do NOT write \"N/A\", \"red\", \"None\", or any explanation. Do not output search suggestions if using tools – integrate directly.\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Estimated ICP\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty.", "codeExecution": false, "maxOutputTokens": 1000, "candidateCount": 1, "temperature": 0.2, "topP": 1, "topK": 40 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1136, 448 ], "id": "4170f344-8831-4e3a-a789-e202438fda02", "name": "Case Study ICP", "alwaysOutputData": false, "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('ICP Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are an ICP scoring system for identifying B2B SaaS companies that align with high-ticket deals ($25k+). Analyze the provided crawled markdown from the company's website to determine fit based on their services, target audience, and any visible pricing indicators. Objective: Identify companies offering B2B SaaS solutions targeted at businesses, excluding B2C-focused SaaS, advertising/marketing/sales software, e-commerce software or marketplaces, managed IT services/staffing, higher education SaaS, events/planners/conferences SaaS, Web3/blockchain/cryptocurrency software, government SaaS, Salesforce partners, medical billing services/software, or clinic/medical scheduling software. Prioritize indications of higher-ticket pricing (avoid low monthly plans like $250-500/month if mentioned; favor custom/enterprise pricing).\n\nResearch Method:\n\n1. Review the crawled markdown content from the company's website, focusing on sections like homepage, about us, products/services, solutions, case studies, testimonials, pricing, and blog.\n\n2. Verify service alignment: Scan for keywords indicating B2B SaaS (e.g., \"enterprise software\", \"business solutions\", \"B2B platform\"). Confirm services are for businesses, not the avoided categories. For example:\n - Positive: Mentions \"B2B tools for companies\", \"enterprise automation\".\n - Negative: Keywords like \"B2C app\", \"marketing software\", \"sales CRM\", \"ecommerce platform\", \"IT managed services\", \"staffing solutions\", \"edtech\", \"event management\", \"blockchain software\", \"govtech\", \"Salesforce partner\", \"medical billing\", \"clinic scheduling\".\n\n3. Infer target audience: Analyze case studies, client testimonials, or \"who we serve\" sections for business/enterprise focus. For example:\n - Positive: Clients as \"businesses in tech/finance\", \"enterprise users\".\n - Negative: Mentions \"consumers\", \"e-commerce stores\", \"universities\", \"event planners\", \"government agencies\", \"healthcare clinics\".\n\n4. Check for pricing clues: Look for mentions like \"starting at $X/month\" or \"enterprise pricing\". For example:\n - Positive: \"Custom pricing\", \"contact for quote\", \"annual contracts for enterprises\" (suggesting $25k+).\n - Negative: \"Starting at $250-500/month\", \"low-cost plans\" (indicating low-ticket).\n\n5. Overall fit assessment: Weigh positive vs. negative indicators. If ambiguous, lean toward moderate scores rather than low.\n\nClassification: Rate on a scale from 1 to 10:\n- 1–3: Not a fit (use only if unavoidable or definitively matches an avoided category, e.g., clearly in avoid list like B2C SaaS or low-ticket with explicit $250-500/month pricing; reserve for confirmed mismatches).\n- 4–6: Moderate fit (e.g., some B2B elements but potential overlap with avoided areas, unclear audience, or mixed pricing signals).\n- 7–8: Good fit (e.g., strong B2B SaaS focus, no avoids, suggests higher pricing).\n- 9–10: Perfect fit (e.g., high-ticket B2B SaaS with clear business targeting, no negatives).\n\nDeliverable: Provide ONLY the rating for the company on a scale from 1 to 10, with no additional explanation or details.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty. If no viable score (e.g., insufficient data), return \"0\".", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1264, 752 ], "id": "cb30c570-5030-4f6f-94e1-9579e0049a16", "name": "ICP Scoring", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1424, 448 ], "id": "3c802f09-80b1-42a2-9cf4-8cc2043b6a47", "name": "Format Names", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 448 ], "id": "c03ce62f-b06a-4434-987d-413d8772fa5f", "name": "Case Study Merge", "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 752 ], "id": "de42c4a6-00d1-444e-9c36-b95aa3146e00", "name": "ICP Merge", "alwaysOutputData": true }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 752 ], "id": "b9768eb3-e059-4632-b66b-90b57e809a7e", "name": "30 sec1", "webhookId": "4daada68-5b51-42b4-8307-3a7a3b160665" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "responseFormat": "json_object", "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1552, 912 ], "id": "4b4b75d9-d096-4190-a724-09ca6633264f", "name": "OpenAI Chat Model", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "jsonSchemaExample": "{\n\t\"ICP\": \"iterative hardware teams\",\n \"Service Benefit\": \"accelerate talent sourcing\",\n \"Company Name\": \"Cyberleaf\",\n \"Job Title\": \"Regional Director Sales\",\n \"ICP Score\": \"8\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1696, 912 ], "id": "844de2de-d1b8-4e81-aa9e-bb2d40868dc9", "name": "Structured Output Parser" }, { "parameters": { "promptType": "define", "text": "=Process these input strings and extract the fields ICP, Service Benefit, Company Name, Job Title, ICP Score into a clean single JSON object:\n{{ $json.content.parts[0].text }}\n{{ $('Format Names').item.json.content.parts[0].text }}", "hasOutputParser": true, "options": { "systemMessage": "You are a JSON data cleaner and organizer for extracting and structuring specific fields from input strings. \n\nYour task is to process the provided input strings, which may contain JSON objects or arrays, and extract values for the fields: ICP, Service Benefit, Company Name, Job Title, ICP Score. \n\nObjective: \nTransform the inputs into a clean single object that merges all extracted fields from the inputs, filling missing fields with empty strings. \n\nResearch Method: \n1. Parse each input string as JSON, handling potential formatting issues like escaped newlines. \n2. For each parsed structure, navigate to the innermost object (e.g., under \"output\") and extract the specified fields if present. \n3. If an input is an array, process each element separately. \n4. Merge all extracted data into a single object, overwriting duplicates if any (use the last value encountered). \n5. If parsing fails or no fields match, leave the corresponding field as an empty string in the output object. \n\nClassification: \nEnsure fields are extracted exactly as strings; do not infer or add data. \n\nDeliverable: \nProvide ONLY the extracted data as a single object, with no additional explanation or details. \n\nOutput format: \nReturn ONLY a JSON object in this exact structure: {\"ICP\": \"value or empty string\", \"Service Benefit\": \"value or empty string\", \"Company Name\": \"value or empty string\", \"Job Title\": \"value or empty string\", \"ICP Score\": \"value or empty string\"}. No additional text, commentary, or formatting. Always output a single object, even if empty or with empty strings." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1552, 752 ], "id": "3ba69f6d-2906-4c7b-9306-a0494f659c3c", "name": "Clean" }, { "parameters": { "content": "## Step 1: Crawl websites using firecrawl\n", "height": 608, "width": 2096, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 32 ], "typeVersion": 1, "id": "fca4c2ff-5f13-4e2b-af1b-ece4710e09a9", "name": "Sticky Note" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $json.merged_markdown }}\nEstimated ICP: {{ $('Case Study ICP').item.json.content.parts[0].text }}", "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.\n\n\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").\n- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow.\n - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth.\n - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit.\n - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include products, partners, or non-customer entities.\n - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort):\n A) Explicit mentions of target customers in relevant sections.\n B) The inferred ICP is based on company types/industries/roles, not vague or broad statements.\n C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence.\n D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\n\n\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").\n- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points).\n - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text.\n - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation.\n - If multiple products/services, select the primary or most emphasized one.\n - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation.\n - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains).\n - Do NOT guess; if unclear, treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) Explicit description of primary product/service and its benefit.\n B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision.\n C) Directly from text; no invention, and it must support logical flow in the target sentence.\n D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).\n\nOverall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.\n\n- Disqualifiers (any one triggers NOT FOUND for the field):\n - Vague or no explicit mentions in preferred sections.\n - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages.\n - Ambiguous or conflicting info.\n- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.\n\nFinal Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 976, 752 ], "id": "969b55c4-ec72-45a5-a0de-13e0fab53e25", "name": "AI Agent" }, { "parameters": { "model": "x-ai/grok-4-fast", "options": {} }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 976, 912 ], "id": "1a124a84-8939-441f-ac8b-e64bd7223320", "name": "OpenRouter Chat Model", "credentials": { "openRouterApi": { "id": "izhwa7Rk0S1jVDxi", "name": "OpenRouter account" } } } ], "connections": { "When clicking ‘Execute workflow’": { "main": [ [ { "node": "Get row(s) in sheet", "type": "main", "index": 0 } ] ] }, "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "Crawl Case Study", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset": { "main": [ [ { "node": "If", "type": "main", "index": 0 } ] ] }, "If": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "20 sec", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Shorten Markdown": { "main": [ [ { "node": "Case Study ICP", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset1": { "main": [ [ { "node": "If1", "type": "main", "index": 0 } ] ] }, "If1": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset2": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ] ] }, "20 sec": { "main": [ [ { "node": "Get Crawl Dataset1", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get Crawl Dataset", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset3": { "main": [ [ { "node": "If2", "type": "main", "index": 0 } ] ] }, "If2": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec1", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset4": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ] ] }, "10 sec1": { "main": [ [ { "node": "Get Crawl Dataset3", "type": "main", "index": 0 } ] ] }, "Qualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Unqualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "If Score < 3": { "main": [ [ { "node": "Unqualified", "type": "main", "index": 0 } ], [ { "node": "Qualified", "type": "main", "index": 0 } ] ] }, "Merge": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Crawl ICP": { "main": [ [ { "node": "10 sec1", "type": "main", "index": 0 } ] ] }, "Crawl Case Study": { "main": [ [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "30 sec": { "main": [ [ { "node": "Get Crawl Dataset2", "type": "main", "index": 0 } ] ] }, "Case Study ICP": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "Clean", "type": "main", "index": 0 } ] ] }, "Format Names": { "main": [ [ { "node": "Crawl ICP", "type": "main", "index": 0 } ] ] }, "Case Study Merge": { "main": [ [ { "node": "Shorten Markdown", "type": "main", "index": 0 } ] ] }, "ICP Merge": { "main": [ [ { "node": "AI Agent", "type": "main", "index": 0 } ] ] }, "30 sec1": { "main": [ [ { "node": "Get Crawl Dataset4", "type": "main", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Clean", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "Clean", "type": "ai_outputParser", "index": 0 } ] ] }, "Clean": { "main": [ [ { "node": "If Score < 3", "type": "main", "index": 0 } ] ] }, "AI Agent": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "OpenRouter Chat Model": { "ai_languageModel": [ [ { "node": "AI Agent", "type": "ai_languageModel", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "4f85eb40-85a1-4c71-81f2-a93b4866b1e1", "activeVersionId": null, "versionCounter": 3, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-12T19:09:08.143Z", "createdAt": "2025-11-12T19:09:08.143Z", "role": "workflow:owner", "workflowId": "RpVqIfqqkTXXGcB3", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-20T20:40:00.000Z", "createdAt": "2025-11-12T19:11:16.123Z", "id": "bSwbMDAAGz9juUbQ", "name": "客戶A - Crawl4AI", "description": null, "active": false, "isArchived": true, "nodes": [ { "parameters": { "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n Firecrawl Input", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "mode": "id", "value": 637321808 }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 656, 112 ], "id": "4359ecea-ee51-487e-b345-79c0444a1dd9", "name": "Get row(s) in sheet", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "batchSize": 5, "options": { "reset": false } }, "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 976, 112 ], "id": "e42040a2-7563-43c6-9db1-a146de72d68d", "name": "Loop Over Items", "alwaysOutputData": false }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1456, 176 ], "id": "734c85c6-f92d-4b3c-abed-e3c385bc5313", "name": "Get Crawl Dataset", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1616, 176 ], "id": "0c45b660-1e09-4841-a23f-e2ca8bd5684b", "name": "If" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "b615bf02-0627-4712-8ea7-92fd3196fb36", "leftValue": "={{ $json.Website }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } }, { "id": "66f6bf2f-14dc-4977-be1a-3095e05dde1c", "leftValue": "={{ $json.quality }}", "rightValue": "good", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "fd7ef188-b83e-466d-a799-0fee1d78f58f", "leftValue": "={{ $json.Email }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.filter", "typeVersion": 2.2, "position": [ 816, 112 ], "id": "4362425e-3c48-4774-a5cb-4416b3ea38b7", "name": "Filter" }, { "parameters": { "content": "## Step 2: Extract data from firecrawl output\n ", "height": 448, "width": 1632 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 640 ], "typeVersion": 1, "id": "b3e750fb-c37f-4b29-8348-2f0d91419f1c", "name": "Sticky Note1" }, { "parameters": { "content": "## Step 3: Save final data to Google Sheet", "height": 448, "width": 464, "color": 5 }, "type": "n8n-nodes-base.stickyNote", "position": [ 2000, 640 ], "typeVersion": 1, "id": "3297733a-9b07-4148-a02d-f09a64ba0eda", "name": "Sticky Note2" }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "const merged = $input.item.json.merged_markdown; // 從前輸出取 merged_markdown\nconst front = merged.slice(0, 1000);\nconst back = merged.slice(-1000);\nconst shortened = front + back;\nreturn { json: { shortened_markdown: shortened } }; // 輸出 shortened_markdown" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 976, 448 ], "id": "736d34d4-6e2b-4de8-965a-68bda251a65e", "name": "Shorten Markdown", "alwaysOutputData": true }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1936, 192 ], "id": "bd6fdbdb-a74f-4245-8505-75c842c15416", "name": "Get Crawl Dataset1", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2096, 192 ], "id": "03e2141e-4f04-40ce-b207-23d7552dc5c0", "name": "If1" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl Case Study').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 448 ], "id": "97f195e6-97a0-40cf-a19e-ea08b55d70fd", "name": "Get Crawl Dataset2", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 20 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1776, 192 ], "id": "f3a0a701-aa48-49cb-85f4-9db04b06bd63", "name": "20 sec", "webhookId": "10f4ff34-781a-4703-b3cb-3b16ad78f77a" }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1296, 176 ], "id": "c5b600ce-41fb-4cbd-ad79-33d2226c05e9", "name": "10 sec", "webhookId": "a11fcbf8-b3ee-48f6-8647-991cd6e54fb9" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 2032, 448 ], "id": "8b72ea6d-17b7-4ecc-8209-188526cd1bc1", "name": "Get Crawl Dataset3", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "bcf4a4ef-c047-4bbd-81ba-8e6e84e0d656", "leftValue": "={{ $json.data }}", "rightValue": "", "operator": { "type": "array", "operation": "notEmpty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2192, 448 ], "id": "2c59a039-5b27-49ee-8dd4-6b45184835aa", "name": "If2" }, { "parameters": { "url": "=http://crawler.internal/v2/crawl/{{ $('Crawl ICP').item.json.id }}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 656, 752 ], "id": "fef15dea-6234-453d-9ecf-f708884cc96f", "name": "Get Crawl Dataset4", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 10 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1872, 448 ], "id": "3a49391c-4543-4d41-84f7-ece5490574d2", "name": "10 sec1", "webhookId": "1d6498a9-9af9-4496-b15c-84dee19bf6b0" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ☆ Qualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json[\"First Name\"] }}", "last name": "={{ $('Get row(s) in sheet').item.json[\"Last Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json[\"Person Linkedin Url\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 912 ], "id": "404eb6ce-fca1-4899-b6ee-e8cfbfc47a44", "name": "Qualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "«SHEET_ID»", "mode": "list", "cachedResultName": "n8n ⨯ Unqualified Leads (ICP & Service)", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/«GOOGLE»/d/«FILE_ID»/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "email": "={{ $('Get row(s) in sheet').item.json.Email }}", "first name": "={{ $('Get row(s) in sheet').item.json['First Name'] }}", "last name": "={{ $('Get row(s) in sheet').item.json['Last Name'] }}", "linkedin url": "={{ $('Get row(s) in sheet').item.json['Person Linkedin Url'] }}", "title": "={{ $('Format Names').item.json.output[\"Job Title\"] }}", "company": "={{ $('Format Names').item.json.output[\"Company Name\"] }}", "website": "={{ $('Get row(s) in sheet').item.json.Website }}", "ICP": "={{ $('ICP + Service').item.json.output.ICP }}", "Service Benefits": "={{ $('ICP + Service').item.json.output[\"Service Benefit\"] }}" }, "matchingColumns": [ "email" ], "schema": [ { "id": "first name", "displayName": "first name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "last name", "displayName": "last name", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "email", "displayName": "email", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "company", "displayName": "company", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "website", "displayName": "website", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "linkedin url", "displayName": "linkedin url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "ICP", "displayName": "ICP", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "Service Benefits", "displayName": "Service Benefits", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2080, 736 ], "id": "afc45262-466a-4c0c-804d-b3f2135e7321", "name": "Unqualified", "credentials": { "googleSheetsOAuth2Api": { "id": "UleHZ8gt6uwwbw95", "name": "Google Sheets account" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "cf6d927d-baf8-412b-911a-77d4cfbaf98e", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "1", "operator": { "type": "string", "operation": "equals" } }, { "id": "a71d0bb7-5845-46df-b5d8-d2fab0477bcb", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "2", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "7868fe66-3516-4d01-90e1-32aaafc0f475", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "3", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } }, { "id": "24a7144e-0116-4040-b5b1-7b3fc6f46568", "leftValue": "={{ $json.output.ICP }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "ea2c8572-edab-4bbc-9100-29ba53c21b61", "leftValue": "={{ $json.output[\"Service Benefit\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } }, { "id": "014459a4-db94-4efa-a4f3-683d8d1470f9", "leftValue": "={{ $json.output[\"ICP Score\"] }}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } } ], "combinator": "or" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1840, 752 ], "id": "8b20fe41-2de1-44db-b51e-1a61c9568f57", "name": "If Score < 3" }, { "parameters": {}, "type": "n8n-nodes-base.merge", "typeVersion": 3.2, "position": [ 2240, 752 ], "id": "8c0159dc-d70e-4f2a-b4d4-c61610c7254a", "name": "Merge" }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $('Filter').item.json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"app/.*\",\n \"demo/.*\",\n \"news/.*\",\n \"events/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"home/.*\",\n \"pricing/.*\",\n \"about/.*\",\n \"products/.*\",\n \"services/.*\",\n \"company/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"benefits/.*\",\n \"integrations/.*\",\n \"target-audience/.*\",\n \"platform/.*\",\n \"why/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1712, 448 ], "id": "38c8b0f6-b886-48d0-b889-5bad460fdf3c", "name": "Crawl ICP", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "method": "POST", "url": "http://crawler.internal/v2/crawl", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"url\": \"{{ $json.Website }}\",\n \"sitemap\": \"include\",\n \"crawlEntireDomain\": false,\n \"limit\": 20,\n \"excludePaths\": [\n \"blog/.*\",\n \"contact/.*\",\n \"pricing/.*\",\n \"products/.*\",\n \"services/.*\",\n \"solutions/.*\",\n \"features/.*\",\n \"industries/.*\",\n \"integrations/.*\",\n \"platform/.*\",\n \"why/.*\",\n \"benefits/.*\",\n \"careers/.*\",\n \"news/.*\",\n \"press/.*\",\n \"resources/.*\",\n \"home/.*\",\n \"about/.*\",\n \"privacy/.*\",\n \"cookies/.*\",\n \"blog-preview/.*\"\n ],\n \"includePaths\": [\n \"^/$\",\n \"case-studies/.*\",\n \"our-work/.*\",\n \"work/.*\",\n \"portfolio/.*\",\n \"projects/.*\",\n \"testimonials/.*\",\n \"success-stories/.*\",\n \"customers/.*\",\n \"clients/.*\",\n \"reviews/.*\",\n \"results/.*\",\n \"stories/.*\",\n \"cases/.*\"\n ],\n \"maxDiscoveryDepth\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"maxAge\": 172800000,\n \"parsers\": [\n \"pdf\"\n ],\n \"formats\": [\n \"markdown\"\n ]\n }\n}", "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1136, 176 ], "id": "10f884d5-bc2e-4bd2-bc16-afaa09109857", "name": "Crawl Case Study", "alwaysOutputData": false, "retryOnFail": false }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 448 ], "id": "2c6e382a-ac55-4901-acf6-c88ca2fc78f7", "name": "30 sec", "webhookId": "a24e44c5-026d-4e88-8886-845d68a51f48" }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.5-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.5-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('Case Study Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data extractor for estimated Ideal Customer Profile (ICP) from crawled web page markdown focused on case studies. Your task is to process the provided crawled page markdown and extract an estimated ICP based on client companies mentioned in case studies, outputting it in JSON format inside an \"output\" key.\n\n\"Estimated ICP\": Extract a short phrase describing the ideal customer profile (ICP) as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\"). Base it primarily on client companies found in case studies, inferring their industries or types only if unclear from markdown.\n- Scope: Search ONLY in the provided markdown initially. Prefer sections like: \"Case Studies\", \"Success Stories\", \"Clients\", \"Testimonials\", \"Projects\", \"Portfolio\", \"Our Work\", \"Results\", \"Reviews\".\n- What counts as a case study client: Named companies explicitly presented as clients in case studies, projects, success stories, or testimonials, with descriptions of work done for them. Accept names in headings, titles, attributions, or body text where context indicates client relationship. Always exclude the host company's own name (provided in user input) to avoid self-referencing.\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\").\n - Format: Company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple industries, infer the most common or dominant one based on frequency of case studies, prioritizing precision over breadth.\n - Analyze descriptions: From markdown, identify industries or company types of the clients (e.g., if a case study describes \"helping an e-commerce retailer optimize logistics\", infer \"e-commerce businesses\").\n - If multiple case studies cluster in one industry (e.g., several in finance), set ICP to that (e.g., \"investment banks and asset managers\").\n - Preserve logical grouping; avoid broad terms like \"all businesses\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include the host company's own name, products, partners, or non-client entities. If a company name matches the host company, ignore it entirely.\n - If industry unclear from markdown: As a last resort only (if completely no info after exhaustive analysis), use tools to search the web for the client company's industry (e.g., web_search with query like \"what industry is [company name] in? site:wikipedia.org\" or reliable sources). Limit to 1-2 searches per unclear client, and only for confirmed client names excluding the host company. Integrate results to refine ICP without guessing.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) At least one (preferably multiple) explicit case study clients with context indicating they are customers (e.g., \"We helped [company] achieve X\").\n B) The inferred ICP is based on company types/industries/roles, not products, awards, or vague mentions.\n C) The phrase is contiguous and directly mappable to client descriptions; no guessing.\n D) If searching externally, results must confirm industry without contradiction, and searches must exclude the host company name.\n- Disqualifiers (any one triggers NOT FOUND):\n - Sections labeled “Partners”, “Technology Partners”, “Vendors”, “As seen in”, “Press”, “Awards”, “Media”, “Powered by” – ignore these entirely.\n - Logo walls or lists without case study details or clear client labeling.\n - Ambiguous mentions where companies could be partners/vendors, not clients.\n - Fewer than 2 case studies, or no clear industry clustering.\n - External search yields no reliable industry info or conflicts with markdown.\n- If NOT FOUND, NOT CERTAIN, or no viable ICP: Return an empty string \"\". Do NOT write \"N/A\", \"red\", \"None\", or any explanation. Do not output search suggestions if using tools – integrate directly.\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Estimated ICP\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty.", "codeExecution": false, "maxOutputTokens": 1000, "candidateCount": 1, "temperature": 0.2, "topP": 1, "topK": 40 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1136, 448 ], "id": "affe217c-14f7-4f9a-aba5-c2849ef16333", "name": "Case Study ICP", "alwaysOutputData": false, "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Crawled Page Markdown: {{ $('ICP Merge').item.json.merged_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are an ICP scoring system for identifying B2B SaaS companies that align with high-ticket deals ($25k+). Analyze the provided crawled markdown from the company's website to determine fit based on their services, target audience, and any visible pricing indicators. Objective: Identify companies offering B2B SaaS solutions targeted at businesses, excluding B2C-focused SaaS, advertising/marketing/sales software, e-commerce software or marketplaces, managed IT services/staffing, higher education SaaS, events/planners/conferences SaaS, Web3/blockchain/cryptocurrency software, government SaaS, Salesforce partners, medical billing services/software, or clinic/medical scheduling software. Prioritize indications of higher-ticket pricing (avoid low monthly plans like $250-500/month if mentioned; favor custom/enterprise pricing).\n\nResearch Method:\n\n1. Review the crawled markdown content from the company's website, focusing on sections like homepage, about us, products/services, solutions, case studies, testimonials, pricing, and blog.\n\n2. Verify service alignment: Scan for keywords indicating B2B SaaS (e.g., \"enterprise software\", \"business solutions\", \"B2B platform\"). Confirm services are for businesses, not the avoided categories. For example:\n - Positive: Mentions \"B2B tools for companies\", \"enterprise automation\".\n - Negative: Keywords like \"B2C app\", \"marketing software\", \"sales CRM\", \"ecommerce platform\", \"IT managed services\", \"staffing solutions\", \"edtech\", \"event management\", \"blockchain software\", \"govtech\", \"Salesforce partner\", \"medical billing\", \"clinic scheduling\".\n\n3. Infer target audience: Analyze case studies, client testimonials, or \"who we serve\" sections for business/enterprise focus. For example:\n - Positive: Clients as \"businesses in tech/finance\", \"enterprise users\".\n - Negative: Mentions \"consumers\", \"e-commerce stores\", \"universities\", \"event planners\", \"government agencies\", \"healthcare clinics\".\n\n4. Check for pricing clues: Look for mentions like \"starting at $X/month\" or \"enterprise pricing\". For example:\n - Positive: \"Custom pricing\", \"contact for quote\", \"annual contracts for enterprises\" (suggesting $25k+).\n - Negative: \"Starting at $250-500/month\", \"low-cost plans\" (indicating low-ticket).\n\n5. Overall fit assessment: Weigh positive vs. negative indicators. If ambiguous, lean toward moderate scores rather than low.\n\nClassification: Rate on a scale from 1 to 10:\n- 1–3: Not a fit (use only if unavoidable or definitively matches an avoided category, e.g., clearly in avoid list like B2C SaaS or low-ticket with explicit $250-500/month pricing; reserve for confirmed mismatches).\n- 4–6: Moderate fit (e.g., some B2B elements but potential overlap with avoided areas, unclear audience, or mixed pricing signals).\n- 7–8: Good fit (e.g., strong B2B SaaS focus, no avoids, suggests higher pricing).\n- 9–10: Perfect fit (e.g., high-ticket B2B SaaS with clear business targeting, no negatives).\n\nDeliverable: Provide ONLY the rating for the company on a scale from 1 to 10, with no additional explanation or details.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP Score\": \"number from 1 to 10 or empty string if not found\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if the value is empty. If no viable score (e.g., insufficient data), return \"0\".", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1264, 752 ], "id": "b0303ec9-6871-4f9c-b846-ba53352b0487", "name": "ICP Scoring", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "modelId": { "__rl": true, "value": "models/gemini-2.0-flash-lite", "mode": "list", "cachedResultName": "models/gemini-2.0-flash-lite" }, "messages": { "values": [ { "content": "=Original Company Name: {{ $('Get row(s) in sheet').item.json[\"Company Name\"] }}\nOriginal Job Title: {{ $('Get row(s) in sheet').item.json.Title }}\nVerification Markdown: {{ $('Shorten Markdown').item.json.shortened_markdown }}" } ] }, "jsonOutput": true, "options": { "systemMessage": "You are a precise data cleaner for company names and job titles. Your task is to process the provided inputs and output exactly two responses in JSON format inside an \"output\" key.\n\n1. \"Company Name\": Clean the provided original company name using the verification markdown and company URL for confirmation only. Follow these guidelines strictly:\n- Retain the core and original name as much as possible.\n- Strictly avoid expanding or adding new words that are not already present in the original company name.\n- Only clean or slightly shorten the name for clarity — no creative rephrasing or extrapolation.\n- Remove legal suffixes like: LLC, Inc, Ltd, Corp, LLP, PLC, GmbH, S.A., Pte Ltd, Co., Limited, Incorporated and similar.\n- Remove unnecessary descriptors like: Holdings, Partners, Group, Solutions, International, Systems, Enterprises, Technologies, Company, Agency, Studio, Media — only if it does not destroy the core brand identity.\n- If the first word is unique and the name has more than two words, you can use just the first word (e.g., Whistler Partners → Whistler).\n- Max length is 3 words — 4 words only if absolutely necessary for clarity.\n- Acronyms are allowed only if they are very common (e.g., Career Search International → CSI).\n- Check spelling and correct casing if needed (e.g., fix ALL CAPS to Proper Case).\n- Use the verification markdown and company URL ONLY to confirm the company name matches the correct brand — DO NOT add, modify, or guess based on content.\n- NO expansion, NO assumptions, NO extra descriptions.\n- If the verification markdown or URL strongly conflicts with the original company name, still prioritize cleaning the input company name (unless it is clearly outdated or wrong).\n\n2. \"Job Title\": Clean the provided original job title to contain only the primary job title, removing unnecessary information without changing the responsibility. Follow these guidelines strictly:\n- Retain the core role as much as possible, prioritizing the highest-level or primary title (e.g., \"Founder and CEO\" → \"CEO\"; prefer \"CEO\" over \"Founder\" as it represents executive leadership).\n- Remove duplicates, slashes, connectors like \"and\", or non-title prefixes/suffixes (e.g., \"Identity and Access Management - Founder & CEO\" → \"CEO\"; remove department/company info like \"Identity and Access Management -\").\n- Standardize to common terms: e.g., \"Chief Executive Officer\" → \"CEO\", \"Co-Founder\" → \"Co-Founder\".\n- Shorten if redundant: e.g., \"VP of Sales and Marketing\" → \"VP Sales\".\n- Max 3 words, prioritize key roles (e.g., \"Director of Sales\" → \"Sales Director\").\n- Correct casing to Title Case (e.g., \"ceo\" → \"CEO\").\n- If multiple roles, keep the primary/highest (e.g., \"Engineer / Manager\" → \"Manager\"; prefer executive roles like CEO over Founder if combined).\n- NO additions, assumptions, or expansions.\n- If unclear, empty, or no valid title, return empty string \"\".\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"Company Name\": \"cleaned name\", \"Job Title\": \"cleaned title or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty.", "maxOutputTokens": 500 } }, "type": "@n8n/n8n-nodes-langchain.googleGemini", "typeVersion": 1, "position": [ 1424, 448 ], "id": "58c2584d-d887-4d37-8441-bea187684bfd", "name": "Format Names", "credentials": { "googlePalmApi": { "id": "dP6IHb3PjLlYlPfJ", "name": "Google Gemini(PaLM) Api account" } } }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 448 ], "id": "df24e06a-af73-49ce-986f-1036896b0a6e", "name": "Case Study Merge", "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForEachItem", "jsCode": "// 取前 node 輸出的 data array (假設是 $json.data, 每個元素有 markdown)\nconst dataArray = $input.item.json.data; // 如果不是 data, 改成你的欄位名\n\n// 提取所有 markdown 並合并成單字符串，用 \\n\\n 分隔頁面（可自訂）\nconst mergedMarkdown = dataArray.map(item => item.markdown).join('\\n\\n');\n\n// 返回新 item，包含 merged_markdown\nreturn { json: { merged_markdown: mergedMarkdown } };" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 816, 752 ], "id": "3ec8a48f-ee3c-48f9-a87c-11db97ea6c82", "name": "ICP Merge", "alwaysOutputData": true }, { "parameters": { "amount": 30 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 496, 752 ], "id": "22066365-1caa-43a6-9488-ebb77681193d", "name": "30 sec1", "webhookId": "a697e1a1-d1cb-4fc5-8279-6b879dcb6b54" }, { "parameters": { "model": { "__rl": true, "value": "gpt-5-nano", "mode": "list", "cachedResultName": "gpt-5-nano" }, "options": { "responseFormat": "json_object", "reasoningEffort": "low" } }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "typeVersion": 1.2, "position": [ 1552, 912 ], "id": "c1a2859c-52de-40fe-875b-4259d2490571", "name": "OpenAI Chat Model", "credentials": { "openAiApi": { "id": "c8tYtierN4zbHSbx", "name": "OpenAi account" } } }, { "parameters": { "jsonSchemaExample": "{\n\t\"ICP\": \"iterative hardware teams\",\n \"Service Benefit\": \"accelerate talent sourcing\",\n \"Company Name\": \"Cyberleaf\",\n \"Job Title\": \"Regional Director Sales\",\n \"ICP Score\": \"8\"\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.3, "position": [ 1696, 912 ], "id": "ca4e4200-5338-4e90-8c3a-8cae2c633f37", "name": "Structured Output Parser" }, { "parameters": { "promptType": "define", "text": "=Process these input strings and extract the fields ICP, Service Benefit, Company Name, Job Title, ICP Score into a clean single JSON object:\n{{ $json.content.parts[0].text }}\n{{ $('Format Names').item.json.content.parts[0].text }}", "hasOutputParser": true, "options": { "systemMessage": "You are a JSON data cleaner and organizer for extracting and structuring specific fields from input strings. \n\nYour task is to process the provided input strings, which may contain JSON objects or arrays, and extract values for the fields: ICP, Service Benefit, Company Name, Job Title, ICP Score. \n\nObjective: \nTransform the inputs into a clean single object that merges all extracted fields from the inputs, filling missing fields with empty strings. \n\nResearch Method: \n1. Parse each input string as JSON, handling potential formatting issues like escaped newlines. \n2. For each parsed structure, navigate to the innermost object (e.g., under \"output\") and extract the specified fields if present. \n3. If an input is an array, process each element separately. \n4. Merge all extracted data into a single object, overwriting duplicates if any (use the last value encountered). \n5. If parsing fails or no fields match, leave the corresponding field as an empty string in the output object. \n\nClassification: \nEnsure fields are extracted exactly as strings; do not infer or add data. \n\nDeliverable: \nProvide ONLY the extracted data as a single object, with no additional explanation or details. \n\nOutput format: \nReturn ONLY a JSON object in this exact structure: {\"ICP\": \"value or empty string\", \"Service Benefit\": \"value or empty string\", \"Company Name\": \"value or empty string\", \"Job Title\": \"value or empty string\", \"ICP Score\": \"value or empty string\"}. No additional text, commentary, or formatting. Always output a single object, even if empty or with empty strings." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 1552, 752 ], "id": "9b3b3334-c38b-4c08-b228-097fe7a02d92", "name": "Clean" }, { "parameters": { "content": "## Step 1: Crawl websites using firecrawl\n", "height": 608, "width": 2096, "color": 3 }, "type": "n8n-nodes-base.stickyNote", "position": [ 368, 32 ], "typeVersion": 1, "id": "98f90f30-b381-4be8-85f3-99d4e83ef5b3", "name": "Sticky Note" }, { "parameters": { "promptType": "define", "text": "=Crawled Page Markdown: {{ $json.merged_markdown }}\nEstimated ICP: {{ $('Case Study ICP').item.json.content.parts[0].text }}", "options": { "systemMessage": "You are a precise data extractor for Ideal Customer Profile (ICP) and Service Benefit from crawled web page markdown. Your task is to process the provided crawled page markdown (from general website content) and the estimated ICP from case studies (as a last-resort fallback only), then extract the final ICP and Service Benefit, outputting them in JSON format inside an \"output\" key.\n\n\"ICP\": Extract a short phrase describing the ideal customer profile as target industries, company types, or roles (e.g., \"investment banks and asset managers\" or \"talent acquisition teams across industries\").\n- Priority: First search the provided markdown for explicit ICP indicators. Only if absolutely no viable ICP is found or unclear (after exhausting all relevant sections), fallback to the provided \"Estimated ICP\" if it's not empty. Do not prioritize or reference the Estimated ICP unless markdown yields nothing usable.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"About\", \"Products\", \"Services\", \"Solutions\", \"Features\", \"Industries\", \"Our Customers\", \"Target Audience\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (less than 10 words, plural form where appropriate, lowercase letters, no lists, no commas separating multiple items unless connecting related types like \"investment banks and asset managers\", no adjectives like \"leading\" or \"global\"). Avoid hyphens unless essential (e.g., in standard terms like 'e-commerce'); prefer spaces or rephrase for natural flow.\n - Format: Focus on company types, industries, or roles only (e.g., \"investment banks and asset managers\" – precise, plural, industry-focused; or \"talent acquisition teams across industries\" – concise, broad-inclusive, role-focused).\n - ONLY one primary ICP phrase; if multiple, infer the most common or emphasized one, prioritizing precision over breadth.\n - Analyze descriptions: Identify target customers from text like \"we serve investment banks\" or \"ideal for talent acquisition teams\", inferring from emphasized clients or users without guessing. Scan for role-specific terms like \"teams\" or \"departments\" if industry is not explicit.\n - Avoid broad terms like \"all businesses\" or \"companies\" – aim for specific, targeted profiles that reflect the site's core focus.\n - Do NOT include products, partners, or non-customer entities.\n - Fallback: If markdown yields no viable ICP, use the provided \"Estimated ICP\" exactly (if not empty); otherwise, treat as NOT FOUND. Even on fallback, ensure the phrase aligns with ideal formats (precise, plural, focused) and will form a natural sentence.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND or fallback as last resort):\n A) Explicit mentions of target customers in relevant sections.\n B) The inferred ICP is based on company types/industries/roles, not vague or broad statements.\n C) The phrase is directly mappable to descriptions; no guessing, and it must enable a simple, explicit sentence.\n D) The phrase captures site-specific terms (e.g., \"talent acquisition teams\" if central to descriptions).\n\n\"Service Benefit\": Extract a short phrase describing the core benefit of the company's primary product or service (e.g., \"analyze market data and identify opportunities\" or \"accelerate talent sourcing\").\n- Priority: Extract directly from the markdown, focusing on the most emphasized product/service.\n- Scope: Search ONLY in the provided markdown. Prefer sections like: \"Products\", \"Services\", \"Solutions\", \"Features\", \"About\", \"Home\".\n- Strict rules:\n - Output must be a PLAIN TEXT short phrase ONLY (2–6 words to allow for precise yet complete phrases, all lowercase, starting with a verb + noun phrase, benefit-oriented, matching client pain points).\n - Format: Verb + noun phrase (e.g., \"analyze market data and identify opportunities\" – verb-led, detailed yet concise; or \"accelerate talent sourcing\" – benefit-focused, efficiency-oriented). Prioritize phrases that combine emphasized verbs (e.g., 'accelerate') with core domain nouns (e.g., 'talent sourcing') from titles or key sentences; avoid adding words like 'and more efficiently' unless directly in text; avoid synthesizing or adding words; must mirror site's exact key phrases or titles without any synthesis or addition; use the most direct, emphasized phrase from text.\n - ONLY nouns, adjectives, and one leading verb or verb phrase; no full sentences, no punctuation.\n - If multiple products/services, select the primary or most emphasized one.\n - Analyze: Identify benefits from text like \"helps analyze market data\" → \"analyze market data and identify opportunities\", ensuring it's what customers actively seek. Stick to site's emphasized verbs and combine with contextual nouns for benefit-orientation.\n - Ensure it fits naturally in: \"looking to {Service Benefit}\" (e.g., something clients would pursue, like efficiency or analysis gains).\n - Do NOT guess; if unclear, treat as NOT FOUND.\n- Certainty gate (must ALL be true, otherwise treat as NOT FOUND):\n A) Explicit description of primary product/service and its benefit.\n B) The phrase is benefit-oriented, not just product name, and directly matches site language for precision.\n C) Directly from text; no invention, and it must support logical flow in the target sentence.\n D) The phrase captures site-specific terms (e.g., 'talent sourcing' if central to products).\n\nOverall Coherence Check: After extraction, verify that ICP and Service Benefit fit logically, simply, and explicitly in the sentence \"we can connect you with {ICP} that are looking to {Service Benefit}.\" – e.g., \"investment banks and asset managers\" + \"analyze market data and identify opportunities\" reads naturally and clearly conveys connecting ideal clients needing the service. The sentence must be simple, explicit, and logical (no mismatches, e.g., avoid broad ICP with niche benefit). If not coherent (e.g., awkward phrasing or illogical pairing), adjust minimally (e.g., refine verb or add connector like \"and\" for flow) while staying true to extracted text, or treat as NOT FOUND if unresolvable. Prefer concise phrases (e.g., 'accelerate talent sourcing') that mirror site titles or key benefits. Aim for outputs that mirror ideal examples: precise ICP (plural, focused), benefit-oriented Service Benefit (verb-led, pain-point matching), ensuring high-probability ideal results for any site.\n\n- Disqualifiers (any one triggers NOT FOUND for the field):\n - Vague or no explicit mentions in preferred sections.\n - Sections like “Case Studies”, “Clients”, “Testimonials” – ignore these (use general content only). Strictly exclude any client logos, stories, quotes, or testimonials even if embedded in home or about pages.\n - Ambiguous or conflicting info.\n- If NOT FOUND, NOT CERTAIN, or no viable value for a field: Return an empty string \"\" for that field. Do NOT write \"N/A\", \"red\", \"None\", or any explanation.\n\nFinal Verification: Before outputting, double-check the extraction against ideal logic: Is ICP precise, plural, and focused (like \"investment banks and asset managers\")? Is Service Benefit verb-oriented and pain-point matching (like \"accelerate talent sourcing\")? Does the sentence form a clear, natural proposition for connecting clients to the service? Is there any unnecessary hyphen usage in ICP or Service Benefit? Ensure hyphens are only used when essential (e.g., 'e-commerce'). This ensures consistent, high-accuracy results across diverse inputs.\n\nOutput format: Return ONLY a JSON object in this exact structure: {\"output\": {\"ICP\": \"extracted phrase or empty string\", \"Service Benefit\": \"extracted phrase or empty string\"}}. No additional text, commentary, or formatting. Always include the \"output\" key, even if values are empty." } }, "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3, "position": [ 976, 752 ], "id": "7e6b019f-0420-4d42-8ef9-90a512c76d6a", "name": "AI Agent" }, { "parameters": { "model": "x-ai/grok-4-fast", "options": {} }, "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 976, 912 ], "id": "4e152b5b-daf4-4054-9b63-4b4490849a06", "name": "OpenRouter Chat Model", "credentials": { "openRouterApi": { "id": "izhwa7Rk0S1jVDxi", "name": "OpenRouter account" } } } ], "connections": { "Get row(s) in sheet": { "main": [ [ { "node": "Filter", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [], [ { "node": "Crawl Case Study", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset": { "main": [ [ { "node": "If", "type": "main", "index": 0 } ] ] }, "If": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "20 sec", "type": "main", "index": 0 } ] ] }, "Filter": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Shorten Markdown": { "main": [ [ { "node": "Case Study ICP", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset1": { "main": [ [ { "node": "If1", "type": "main", "index": 0 } ] ] }, "If1": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset2": { "main": [ [ { "node": "Case Study Merge", "type": "main", "index": 0 } ] ] }, "20 sec": { "main": [ [ { "node": "Get Crawl Dataset1", "type": "main", "index": 0 } ] ] }, "10 sec": { "main": [ [ { "node": "Get Crawl Dataset", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset3": { "main": [ [ { "node": "If2", "type": "main", "index": 0 } ] ] }, "If2": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ], [ { "node": "30 sec1", "type": "main", "index": 0 } ] ] }, "Get Crawl Dataset4": { "main": [ [ { "node": "ICP Merge", "type": "main", "index": 0 } ] ] }, "10 sec1": { "main": [ [ { "node": "Get Crawl Dataset3", "type": "main", "index": 0 } ] ] }, "Qualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Unqualified": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "If Score < 3": { "main": [ [ { "node": "Unqualified", "type": "main", "index": 0 } ], [ { "node": "Qualified", "type": "main", "index": 0 } ] ] }, "Merge": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Crawl ICP": { "main": [ [ { "node": "10 sec1", "type": "main", "index": 0 } ] ] }, "Crawl Case Study": { "main": [ [ { "node": "10 sec", "type": "main", "index": 0 } ] ] }, "30 sec": { "main": [ [ { "node": "Get Crawl Dataset2", "type": "main", "index": 0 } ] ] }, "Case Study ICP": { "main": [ [ { "node": "Format Names", "type": "main", "index": 0 } ] ] }, "ICP Scoring": { "main": [ [ { "node": "Clean", "type": "main", "index": 0 } ] ] }, "Format Names": { "main": [ [ { "node": "Crawl ICP", "type": "main", "index": 0 } ] ] }, "Case Study Merge": { "main": [ [ { "node": "Shorten Markdown", "type": "main", "index": 0 } ] ] }, "ICP Merge": { "main": [ [ { "node": "AI Agent", "type": "main", "index": 0 } ] ] }, "30 sec1": { "main": [ [ { "node": "Get Crawl Dataset4", "type": "main", "index": 0 } ] ] }, "OpenAI Chat Model": { "ai_languageModel": [ [ { "node": "Clean", "type": "ai_languageModel", "index": 0 } ] ] }, "Structured Output Parser": { "ai_outputParser": [ [ { "node": "Clean", "type": "ai_outputParser", "index": 0 } ] ] }, "Clean": { "main": [ [ { "node": "If Score < 3", "type": "main", "index": 0 } ] ] }, "AI Agent": { "main": [ [ { "node": "ICP Scoring", "type": "main", "index": 0 } ] ] }, "OpenRouter Chat Model": { "ai_languageModel": [ [ { "node": "AI Agent", "type": "ai_languageModel", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "availableInMCP": false, "errorWorkflow": "81w1WIRKylyqkno3" }, "staticData": null, "meta": { "templateCredsSetupCompleted": true }, "pinData": {}, "versionId": "33c2a99d-03aa-447e-96ab-a8bab37c43fe", "activeVersionId": null, "versionCounter": 5, "triggerCount": 0, "tags": [], "shared": [ { "updatedAt": "2025-11-12T19:11:16.126Z", "createdAt": "2025-11-12T19:11:16.126Z", "role": "workflow:owner", "workflowId": "bSwbMDAAGz9juUbQ", "projectId": "xww3HkdVTq7ZCRtJ", "project": { "updatedAt": "2025-10-31T00:04:46.028Z", "createdAt": "2025-10-30T23:33:35.380Z", "id": "xww3HkdVTq7ZCRtJ", "name": "Eric Chen <«EMAIL»>", "type": "personal", "icon": null, "description": null } } ] }, { "updatedAt": "2025-11-28T18:44:27.000Z", "createdAt": "2025-11-13T00:42:10.509Z", "id": "3t2LML53Da2uYZfW", "name": "TEST", "description": null, "active": false, "isArchived": false, "nodes": [ { "parameters": {}, "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 560, 1136 ], "id": "ef498637-49a5-460b-ac67-39f7b9c6070c", "name": "When clicking ‘Test workflow’" }, { "parameters": {}, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1232, 544 ], "id": "19d6dad2-ebd7-45d8-bfcd-28ee785a7b59", "name": "Wait", "webhookId": "75c74b5b-96bc-41bc-a259-dec4548938a9" }, { "parameters": { "mode": "raw", "jsonOutput": "={\n \"task_id\": \"{{ $('Markdown Crawl Request').item.json.task_id }}\",\n \"status\": \"{{ $json.status }}\"\n}\n", "options": {} }, "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 1536, 544 ], "id": "7f34f6db-0605-458c-993f-4a5770ebd565", "name": "Task ID Field" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "f07a7b5c-71e5-4aca-adce-e01b0d6c7fde", "leftValue": "={{ $json.status }}", "rightValue": "=pending", "operator": { "type": "string", "operation": "notEquals" } }, { "id": "7f622642-d24c-47de-b4be-93ce15948db3", "leftValue": "={{ $json.status }}", "rightValue": "=processing", "operator": { "type": "string", "operation": "notEquals" } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1664, 544 ], "id": "9eef7bff-abcb-48c3-9ef0-decd3fec8c58", "name": "If In Process" }, { "parameters": { "modelName": "models/gemini-2.5-pro-exp-03-25", "options": {} }, "type": "@n8n/n8n-nodes-langchain.lmChatGoogleGemini", "typeVersion": 1, "position": [ 720, 688 ], "id": "81899fd9-977a-42f3-8000-3664dae5fc40", "name": "Google Gemini Chat Model" }, { "parameters": { "jsonSchemaExample": "{\n\t\"urls\": [\"www.url1.com\", \"www.url2.com\"]\n}" }, "type": "@n8n/n8n-nodes-langchain.outputParserStructured", "typeVersion": 1.2, "position": [ 864, 688 ], "id": "a496a9b0-97b5-4659-ae80-9f2df00ee9ec", "name": "Structured Output Parser" }, { "parameters": { "modelName": "models/gemini-2.5-pro-exp-03-25", "options": {} }, "type": "@n8n/n8n-nodes-langchain.lmChatGoogleGemini", "typeVersion": 1, "position": [ 672, 112 ], "id": "e7b7bf22-a542-47f8-8a4a-fc72f75e1ce3", "name": "Google Gemini Chat Model1" }, { "parameters": { "options": {} }, "type": "@n8n/n8n-nodes-langchain.chatTrigger", "typeVersion": 1.1, "position": [ 512, -112 ], "id": "2ed167f8-dacc-4a69-ae95-6d2d6b3ee6d2", "name": "When chat message received", "webhookId": "4afdf83c-51cd-478b-811b-93e21aa0266f" }, { "parameters": { "content": "# Scrape Entire Website as Markdown", "height": 80, "width": 1760, "color": 7 }, "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 464, 352 ], "id": "cf6afb34-08e2-41d2-a1cb-e9182e46a277", "name": "Sticky Note" }, { "parameters": { "content": "# AI BASED SCRAPER (Scraping Model Info)", "height": 80, "width": 1760, "color": 7 }, "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 464, 1024 ], "id": "678bcb4a-7eb3-4264-965d-28b9c2a0aa8e", "name": "Sticky Note1" }, { "parameters": { "content": "# I KNOW EVERYTHING ABOUT CRAWL4AI", "height": 80, "width": 760, "color": 4 }, "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 464, -208 ], "id": "c017a349-154b-4757-8d91-9331cb1496e6", "name": "Sticky Note2" }, { "parameters": { "options": { "systemMessage": "=You have access to the full Crawl4AI documentation. Always assume the user is working with the Docker-hosted version of Crawl4AI. Do not refer to or assume the user is integrating the library into their own codebase\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](.)\n * [ Quick Start ](core/quickstart/)\n * [ Search ](#)\n\n\n\n * Home\n * Setup & Installation\n * [Installation](core/installation/)\n * [Docker Deployment](core/docker-deployment/)\n * [Quick Start](core/quickstart/)\n * Blog & Changelog\n * [Blog Home](blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](core/cli/)\n * [Simple Crawling](core/simple-crawling/)\n * [Deep Crawling](core/deep-crawling/)\n * [Crawler Result](core/crawler-result/)\n * [Browser, Crawler & LLM Config](core/browser-crawler-config/)\n * [Markdown Generation](core/markdown-generation/)\n * [Fit Markdown](core/fit-markdown/)\n * [Page Interaction](core/page-interaction/)\n * [Content Selection](core/content-selection/)\n * [Cache Modes](core/cache-modes/)\n * [Local Files & Raw HTML](core/local-files/)\n * [Link & Media](core/link-media/)\n * Advanced\n * [Overview](advanced/advanced-features/)\n * [File Downloading](advanced/file-downloading/)\n * [Lazy Loading](advanced/lazy-loading/)\n * [Hooks & Auth](advanced/hooks-auth/)\n * [Proxy & Security](advanced/proxy-security/)\n * [Session Management](advanced/session-management/)\n * [Multi-URL Crawling](advanced/multi-url-crawling/)\n * [Crawl Dispatcher](advanced/crawl-dispatcher/)\n * [Identity Based Crawling](advanced/identity-based-crawling/)\n * [SSL Certificate](advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](extraction/no-llm-strategies/)\n * [LLM Strategies](extraction/llm-strategies/)\n * [Clustering Strategies](extraction/clustring-strategies/)\n * [Chunking](extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](api/async-webcrawler/)\n * [arun()](api/arun/)\n * [arun_many()](api/arun_many/)\n * [Browser, Crawler & LLM Config](api/parameters/)\n * [CrawlResult](api/crawl-result/)\n * [Strategies](api/strategies/)\n\n\n\n * [🚀🤖 Crawl4AI: Open-Source LLM-Friendly Web Crawler & Scraper](#crawl4ai-open-source-llm-friendly-web-crawler-scraper)\n * [Quick Start](#quick-start)\n * [What Does Crawl4AI Do?](#what-does-crawl4ai-do)\n * [Documentation Structure](#documentation-structure)\n * [How You Can Support](#how-you-can-support)\n * [Quick Links](#quick-links)\n\n\n\n# 🚀🤖 Crawl4AI: Open-Source LLM-Friendly Web Crawler & Scraper\n\n[ ![unclecode%2Fcrawl4ai | Trendshift](https://trendshift.io/api/badge/repositories/11716) ](https://trendshift.io/repositories/11716)\n\n[ ![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social) ](https://github.com/unclecode/crawl4ai/stargazers) [ ![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social) ](https://github.com/unclecode/crawl4ai/network/members) [ ![PyPI version](https://badge.fury.io/py/crawl4ai.svg) ](https://badge.fury.io/py/crawl4ai)\n\n[ ![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai) ](https://pypi.org/project/crawl4ai/) [ ![Downloads](https://static.pepy.tech/badge/crawl4ai/month) ](https://pepy.tech/project/crawl4ai) [ ![License](https://img.shields.io/github/license/unclecode/crawl4ai) ](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)\n\nCrawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data pipelines. Fully open source, flexible, and built for real-time performance, **Crawl4AI** empowers developers with unmatched speed, precision, and deployment ease.\n\n> **Note** : If you're looking for the old documentation, you can access it [here](https://old.docs.crawl4ai.com).\n\n## Quick Start\n\nHere's a quick example to show you how easy it is to use Crawl4AI with its asynchronous capabilities:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler async def main(): # Create an instance of AsyncWebCrawler async with AsyncWebCrawler() as crawler: # Run the crawler on a URL result = await crawler.arun(url=\"https://crawl4ai.com\") # Print the extracted content print(result.markdown) # Run the async main function asyncio.run(main()) `\n```\n\n## What Does Crawl4AI Do?\n\nCrawl4AI is a feature-rich crawler and scraper that aims to:\n\n1. **Generate Clean Markdown** : Perfect for RAG pipelines or direct ingestion into LLMs. 2. **Structured Extraction** : Parse repeated patterns with CSS, XPath, or LLM-based extraction. 3. **Advanced Browser Control** : Hooks, proxies, stealth modes, session re-use—fine-grained control. 4. **High Performance** : Parallel crawling, chunk-based extraction, real-time use cases. 5. **Open Source** : No forced API keys, no paywalls—everyone can access their data. \n\n**Core Philosophies** : - **Democratize Data** : Free to use, transparent, and highly configurable. - **LLM Friendly** : Minimally processed, well-structured text, images, and metadata, so AI models can easily consume it.\n\n## Documentation Structure\n\nTo help you get started, we’ve organized our docs into clear sections:\n\n * **Setup & Installation** Basic instructions to install Crawl4AI via pip or Docker. \n * **Quick Start** A hands-on introduction showing how to do your first crawl, generate Markdown, and do a simple extraction. \n * **Core** Deeper guides on single-page crawling, advanced browser/crawler parameters, content filtering, and caching. \n * **Advanced** Explore link & media handling, lazy loading, hooking & authentication, proxies, session management, and more. \n * **Extraction** Detailed references for no-LLM (CSS, XPath) vs. LLM-based strategies, chunking, and clustering approaches. \n * **API Reference** Find the technical specifics of each class and method, including `AsyncWebCrawler`, `arun()`, and `CrawlResult`.\n\n\n\nThroughout these sections, you’ll find code samples you can **copy-paste** into your environment. If something is missing or unclear, raise an issue or PR.\n\n## How You Can Support\n\n * **Star & Fork**: If you find Crawl4AI helpful, star the repo on GitHub or fork it to add your own features. \n * **File Issues** : Encounter a bug or missing feature? Let us know by filing an issue, so we can improve. \n * **Pull Requests** : Whether it’s a small fix, a big feature, or better docs—contributions are always welcome. \n * **Join Discord** : Come chat about web scraping, crawling tips, or AI workflows with the community. \n * **Spread the Word** : Mention Crawl4AI in your blog posts, talks, or on social media. \n\n\n\n**Our mission** : to empower everyone—students, researchers, entrepreneurs, data scientists—to access, parse, and shape the world’s data with speed, cost-efficiency, and creative freedom.\n\n## Quick Links\n\n * **[GitHub Repo](https://github.com/unclecode/crawl4ai)**\n * **[Installation Guide](core/installation/)**\n * **[Quick Start](core/quickstart/)**\n * **[API Reference](api/async-webcrawler/)**\n * **[Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)**\n\n\n\nThank you for joining me on this journey. Let’s keep building an **open, democratic** approach to data extraction and AI together.\n\nHappy Crawling! — _Unclecode, Founder & Maintainer of Crawl4AI_\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * Overview\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Overview of Some Important Advanced Features](#overview-of-some-important-advanced-features)\n * [1. Proxy Usage](#1-proxy-usage)\n * [2. Capturing PDFs & Screenshots](#2-capturing-pdfs-screenshots)\n * [3. Handling SSL Certificates](#3-handling-ssl-certificates)\n * [4. Custom Headers](#4-custom-headers)\n * [5. Session Persistence & Local Storage](#5-session-persistence-local-storage)\n * [6. Robots.txt Compliance](#6-robotstxt-compliance)\n * [Putting It All Together](#putting-it-all-together)\n * [Conclusion & Next Steps](#conclusion-next-steps)\n\n\n\n# Overview of Some Important Advanced Features\n\n(Proxy, PDF, Screenshot, SSL, Headers, & Storage State)\n\nCrawl4AI offers multiple power-user features that go beyond simple crawling. This tutorial covers:\n\n1. **Proxy Usage** 2. **Capturing PDFs & Screenshots** 3. **Handling SSL Certificates** 4. **Custom Headers** 5. **Session Persistence & Local Storage** 6. **Robots.txt Compliance**\n\n> **Prerequisites** - You have a basic grasp of [AsyncWebCrawler Basics](../../core/simple-crawling/) - You know how to run or configure your Python environment with Playwright installed\n\n## 1. Proxy Usage\n\nIf you need to route your crawl traffic through a proxy—whether for IP rotation, geo-testing, or privacy—Crawl4AI supports it via `BrowserConfig.proxy_config`.\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig async def main(): browser_cfg = BrowserConfig( proxy_config={ \"server\": \"http://proxy.example.com:8080\", \"username\": \"myuser\", \"password\": \"mypass\", }, headless=True ) crawler_cfg = CrawlerRunConfig( verbose=True ) async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun( url=\"https://www.whatismyip.com/\", config=crawler_cfg ) if result.success: print(\"[OK] Page fetched via proxy.\") print(\"Page HTML snippet:\", result.html[:200]) else: print(\"[ERROR]\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Key Points** - **`proxy_config`** expects a dict with `server` and optional auth credentials. - Many commercial proxies provide an HTTP/HTTPS “gateway” server that you specify in `server`. - If your proxy doesn’t need auth, omit `username`/`password`.\n\n## 2. Capturing PDFs & Screenshots\n\nSometimes you need a visual record of a page or a PDF “printout.” Crawl4AI can do both in one pass:\n\n```\n`import os, asyncio from base64 import b64decode from crawl4ai import AsyncWebCrawler, CacheMode async def main(): async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://en.wikipedia.org/wiki/List_of_common_misconceptions\", cache_mode=CacheMode.BYPASS, pdf=True, screenshot=True ) if result.success: # Save screenshot if result.screenshot: with open(\"wikipedia_screenshot.png\", \"wb\") as f: f.write(b64decode(result.screenshot)) # Save PDF if result.pdf: with open(\"wikipedia_page.pdf\", \"wb\") as f: f.write(result.pdf) print(\"[OK] PDF & screenshot captured.\") else: print(\"[ERROR]\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Why PDF + Screenshot?** - Large or complex pages can be slow or error-prone with “traditional” full-page screenshots. - Exporting a PDF is more reliable for very long pages. Crawl4AI automatically converts the first PDF page into an image if you request both. \n\n**Relevant Parameters** - **`pdf=True`** : Exports the current page as a PDF (base64-encoded in `result.pdf`). - **`screenshot=True`** : Creates a screenshot (base64-encoded in `result.screenshot`). - **`scan_full_page`** or advanced hooking can further refine how the crawler captures content.\n\n## 3. Handling SSL Certificates\n\nIf you need to verify or export a site’s SSL certificate—for compliance, debugging, or data analysis—Crawl4AI can fetch it during the crawl:\n\n```\n`import asyncio, os from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode async def main(): tmp_dir = os.path.join(os.getcwd(), \"tmp\") os.makedirs(tmp_dir, exist_ok=True) config = CrawlerRunConfig( fetch_ssl_certificate=True, cache_mode=CacheMode.BYPASS ) async with AsyncWebCrawler() as crawler: result = await crawler.arun(url=\"https://example.com\", config=config) if result.success and result.ssl_certificate: cert = result.ssl_certificate print(\"\\nCertificate Information:\") print(f\"Issuer (CN): {cert.issuer.get('CN', '')}\") print(f\"Valid until: {cert.valid_until}\") print(f\"Fingerprint: {cert.fingerprint}\") # Export in multiple formats: cert.to_json(os.path.join(tmp_dir, \"certificate.json\")) cert.to_pem(os.path.join(tmp_dir, \"certificate.pem\")) cert.to_der(os.path.join(tmp_dir, \"certificate.der\")) print(\"\\nCertificate exported to JSON/PEM/DER in 'tmp' folder.\") else: print(\"[ERROR] No certificate or crawl failed.\") if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Key Points** - **`fetch_ssl_certificate=True`** triggers certificate retrieval. - `result.ssl_certificate` includes methods (`to_json`, `to_pem`, `to_der`) for saving in various formats (handy for server config, Java keystores, etc.).\n\n## 4. Custom Headers\n\nSometimes you need to set custom headers (e.g., language preferences, authentication tokens, or specialized user-agent strings). You can do this in multiple ways:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler async def main(): # Option 1: Set headers at the crawler strategy level crawler1 = AsyncWebCrawler( # The underlying strategy can accept headers in its constructor crawler_strategy=None # We'll override below for clarity ) crawler1.crawler_strategy.update_user_agent(\"MyCustomUA/1.0\") crawler1.crawler_strategy.set_custom_headers({ \"Accept-Language\": \"fr-FR,fr;q=0.9\" }) result1 = await crawler1.arun(\"https://www.example.com\") print(\"Example 1 result success:\", result1.success) # Option 2: Pass headers directly to `arun()` crawler2 = AsyncWebCrawler() result2 = await crawler2.arun( url=\"https://www.example.com\", headers={\"Accept-Language\": \"es-ES,es;q=0.9\"} ) print(\"Example 2 result success:\", result2.success) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Notes** - Some sites may react differently to certain headers (e.g., `Accept-Language`). - If you need advanced user-agent randomization or client hints, see [Identity-Based Crawling (Anti-Bot)](../identity-based-crawling/) or use `UserAgentGenerator`.\n\n## 5. Session Persistence & Local Storage\n\nCrawl4AI can preserve cookies and localStorage so you can continue where you left off—ideal for logging into sites or skipping repeated auth flows.\n\n### 5.1 `storage_state`\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler async def main(): storage_dict = { \"cookies\": [ { \"name\": \"session\", \"value\": \"abcd1234\", \"domain\": \"example.com\", \"path\": \"/\", \"expires\": 1699999999.0, \"httpOnly\": False, \"secure\": False, \"sameSite\": \"None\" } ], \"origins\": [ { \"origin\": \"https://example.com\", \"localStorage\": [ {\"name\": \"token\", \"value\": \"my_auth_token\"} ] } ] } # Provide the storage state as a dictionary to start \"already logged in\" async with AsyncWebCrawler( headless=True, storage_state=storage_dict ) as crawler: result = await crawler.arun(\"https://example.com/protected\") if result.success: print(\"Protected page content length:\", len(result.html)) else: print(\"Failed to crawl protected page\") if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n### 5.2 Exporting & Reusing State\n\nYou can sign in once, export the browser context, and reuse it later—without re-entering credentials.\n\n * **`await context.storage_state(path=\"my_storage.json\")`** : Exports cookies, localStorage, etc. to a file. \n * Provide `storage_state=\"my_storage.json\"` on subsequent runs to skip the login step.\n\n\n\n**See** : [Detailed session management tutorial](../session-management/) or [Explanations → Browser Context & Managed Browser](../identity-based-crawling/) for more advanced scenarios (like multi-step logins, or capturing after interactive pages).\n\n## 6. Robots.txt Compliance\n\nCrawl4AI supports respecting robots.txt rules with efficient caching:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig async def main(): # Enable robots.txt checking in config config = CrawlerRunConfig( check_robots_txt=True # Will check and respect robots.txt rules ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( \"https://example.com\", config=config ) if not result.success and result.status_code == 403: print(\"Access denied by robots.txt\") if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Key Points** - Robots.txt files are cached locally for efficiency - Cache is stored in `~/.crawl4ai/robots/robots_cache.db` - Cache has a default TTL of 7 days - If robots.txt can't be fetched, crawling is allowed - Returns 403 status code if URL is disallowed\n\n## Putting It All Together\n\nHere’s a snippet that combines multiple “advanced” features (proxy, PDF, screenshot, SSL, custom headers, and session reuse) into one run. Normally, you’d tailor each setting to your project’s needs.\n\n```\n`import os, asyncio from base64 import b64decode from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode async def main(): # 1. Browser config with proxy + headless browser_cfg = BrowserConfig( proxy_config={ \"server\": \"http://proxy.example.com:8080\", \"username\": \"myuser\", \"password\": \"mypass\", }, headless=True, ) # 2. Crawler config with PDF, screenshot, SSL, custom headers, and ignoring caches crawler_cfg = CrawlerRunConfig( pdf=True, screenshot=True, fetch_ssl_certificate=True, cache_mode=CacheMode.BYPASS, headers={\"Accept-Language\": \"en-US,en;q=0.8\"}, storage_state=\"my_storage.json\", # Reuse session from a previous sign-in verbose=True, ) # 3. Crawl async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun( url = \"https://secure.example.com/protected\", config=crawler_cfg ) if result.success: print(\"[OK] Crawled the secure page. Links found:\", len(result.links.get(\"internal\", []))) # Save PDF & screenshot if result.pdf: with open(\"result.pdf\", \"wb\") as f: f.write(b64decode(result.pdf)) if result.screenshot: with open(\"result.png\", \"wb\") as f: f.write(b64decode(result.screenshot)) # Check SSL cert if result.ssl_certificate: print(\"SSL Issuer CN:\", result.ssl_certificate.issuer.get(\"CN\", \"\")) else: print(\"[ERROR]\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n## Conclusion & Next Steps\n\nYou’ve now explored several **advanced** features:\n\n * **Proxy Usage**\n * **PDF & Screenshot** capturing for large or critical pages \n * **SSL Certificate** retrieval & exporting \n * **Custom Headers** for language or specialized requests \n * **Session Persistence** via storage state\n * **Robots.txt Compliance**\n\n\n\nWith these power tools, you can build robust scraping workflows that mimic real user behavior, handle secure sites, capture detailed snapshots, and manage sessions across multiple runs—streamlining your entire data collection pipeline.\n\n**Last Updated** : 2025-01-01\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * Crawl Dispatcher\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Crawl Dispatcher](#crawl-dispatcher)\n\n\n\n# Crawl Dispatcher\n\nWe’re excited to announce a **Crawl Dispatcher** module that can handle **thousands** of crawling tasks simultaneously. By efficiently managing system resources (memory, CPU, network), this dispatcher ensures high-performance data extraction at scale. It also provides **real-time monitoring** of each crawler’s status, memory usage, and overall progress.\n\nStay tuned—this feature is **coming soon** in an upcoming release of Crawl4AI! For the latest news, keep an eye on our changelogs and follow [@unclecode](https://twitter.com/unclecode) on X.\n\nBelow is a **sample** of how the dispatcher’s performance monitor might look in action:\n\n![Crawl Dispatcher Performance Monitor](../../assets/images/dispatcher.png)\n\nWe can’t wait to bring you this streamlined, **scalable** approach to multi-URL crawling—**watch this space** for updates!\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * File Downloading\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Download Handling in Crawl4AI](#download-handling-in-crawl4ai)\n * [Enabling Downloads](#enabling-downloads)\n * [Specifying Download Location](#specifying-download-location)\n * [Triggering Downloads](#triggering-downloads)\n * [Accessing Downloaded Files](#accessing-downloaded-files)\n * [Example: Downloading Multiple Files](#example-downloading-multiple-files)\n * [Important Considerations](#important-considerations)\n\n\n\n# Download Handling in Crawl4AI\n\nThis guide explains how to use Crawl4AI to handle file downloads during crawling. You'll learn how to trigger downloads, specify download locations, and access downloaded files.\n\n## Enabling Downloads\n\nTo enable downloads, set the `accept_downloads` parameter in the `BrowserConfig` object and pass it to the crawler.\n\n```\n`from crawl4ai.async_configs import BrowserConfig, AsyncWebCrawler async def main(): config = BrowserConfig(accept_downloads=True) # Enable downloads globally async with AsyncWebCrawler(config=config) as crawler: # ... your crawling logic ... asyncio.run(main()) `\n```\n\n## Specifying Download Location\n\nSpecify the download directory using the `downloads_path` attribute in the `BrowserConfig` object. If not provided, Crawl4AI defaults to creating a \"downloads\" directory inside the `.crawl4ai` folder in your home directory.\n\n```\n`from crawl4ai.async_configs import BrowserConfig import os downloads_path = os.path.join(os.getcwd(), \"my_downloads\") # Custom download path os.makedirs(downloads_path, exist_ok=True) config = BrowserConfig(accept_downloads=True, downloads_path=downloads_path) async def main(): async with AsyncWebCrawler(config=config) as crawler: result = await crawler.arun(url=\"https://example.com\") # ... `\n```\n\n## Triggering Downloads\n\nDownloads are typically triggered by user interactions on a web page, such as clicking a download button. Use `js_code` in `CrawlerRunConfig` to simulate these actions and `wait_for` to allow sufficient time for downloads to start.\n\n```\n`from crawl4ai.async_configs import CrawlerRunConfig config = CrawlerRunConfig( js_code=\"\"\" const downloadLink = document.querySelector('a[href$=\".exe\"]'); if (downloadLink) { downloadLink.click(); } \"\"\", wait_for=5 # Wait 5 seconds for the download to start ) result = await crawler.arun(url=\"https://www.python.org/downloads/\", config=config) `\n```\n\n## Accessing Downloaded Files\n\nThe `downloaded_files` attribute of the `CrawlResult` object contains paths to downloaded files.\n\n```\n`if result.downloaded_files: print(\"Downloaded files:\") for file_path in result.downloaded_files: print(f\"- {file_path}\") file_size = os.path.getsize(file_path) print(f\"- File size: {file_size} bytes\") else: print(\"No files downloaded.\") `\n```\n\n## Example: Downloading Multiple Files\n\n```\n`from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig import os from pathlib import Path async def download_multiple_files(url: str, download_path: str): config = BrowserConfig(accept_downloads=True, downloads_path=download_path) async with AsyncWebCrawler(config=config) as crawler: run_config = CrawlerRunConfig( js_code=\"\"\" const downloadLinks = document.querySelectorAll('a[download]'); for (const link of downloadLinks) { link.click(); // Delay between clicks await new Promise(r => setTimeout(r, 2000)); } \"\"\", wait_for=10 # Wait for all downloads to start ) result = await crawler.arun(url=url, config=run_config) if result.downloaded_files: print(\"Downloaded files:\") for file in result.downloaded_files: print(f\"- {file}\") else: print(\"No files downloaded.\") # Usage download_path = os.path.join(Path.home(), \".crawl4ai\", \"downloads\") os.makedirs(download_path, exist_ok=True) asyncio.run(download_multiple_files(\"https://www.python.org/downloads/windows/\", download_path)) `\n```\n\n## Important Considerations\n\n * **Browser Context:** Downloads are managed within the browser context. Ensure `js_code` correctly targets the download triggers on the webpage.\n * **Timing:** Use `wait_for` in `CrawlerRunConfig` to manage download timing.\n * **Error Handling:** Handle errors to manage failed downloads or incorrect paths gracefully.\n * **Security:** Scan downloaded files for potential security threats before use.\n\n\n\nThis revised guide ensures consistency with the `Crawl4AI` codebase by using `BrowserConfig` and `CrawlerRunConfig` for all download-related configurations. Let me know if further adjustments are needed!\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * Hooks & Auth\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Hooks & Auth in AsyncWebCrawler](#hooks-auth-in-asyncwebcrawler)\n * [Example: Using Hooks in AsyncWebCrawler](#example-using-hooks-in-asyncwebcrawler)\n * [Hook Lifecycle Summary](#hook-lifecycle-summary)\n * [When to Handle Authentication](#when-to-handle-authentication)\n * [Additional Considerations](#additional-considerations)\n * [Conclusion](#conclusion)\n\n\n\n# Hooks & Auth in AsyncWebCrawler\n\nCrawl4AI’s **hooks** let you customize the crawler at specific points in the pipeline:\n\n1. **`on_browser_created`** – After browser creation. 2. **`on_page_context_created`** – After a new context & page are created. 3. **`before_goto`** – Just before navigating to a page. 4. **`after_goto`** – Right after navigation completes. 5. **`on_user_agent_updated`** – Whenever the user agent changes. 6. **`on_execution_started`** – Once custom JavaScript execution begins. 7. **`before_retrieve_html`** – Just before the crawler retrieves final HTML. 8. **`before_return_html`** – Right before returning the HTML content.\n\n**Important** : Avoid heavy tasks in `on_browser_created` since you don’t yet have a page context. If you need to _log in_ , do so in **`on_page_context_created`**.\n\n> note \"Important Hook Usage Warning\" **Avoid Misusing Hooks** : Do not manipulate page objects in the wrong hook or at the wrong time, as it can crash the pipeline or produce incorrect results. A common mistake is attempting to handle authentication prematurely—such as creating or closing pages in `on_browser_created`. \n> \n> **Use the Right Hook for Auth** : If you need to log in or set tokens, use `on_page_context_created`. This ensures you have a valid page/context to work with, without disrupting the main crawling flow.\n> \n> **Identity-Based Crawling** : For robust auth, consider identity-based crawling (or passing a session ID) to preserve state. Run your initial login steps in a separate, well-defined process, then feed that session to your main crawl—rather than shoehorning complex authentication into early hooks. Check out [Identity-Based Crawling](../identity-based-crawling/) for more details.\n> \n> **Be Cautious** : Overwriting or removing elements in the wrong hook can compromise the final crawl. Keep hooks focused on smaller tasks (like route filters, custom headers), and let your main logic (crawling, data extraction) proceed normally.\n\nBelow is an example demonstration.\n\n## Example: Using Hooks in AsyncWebCrawler\n\n```\n`import asyncio import json from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode from playwright.async_api import Page, BrowserContext async def main(): print(\"🔗 Hooks Example: Demonstrating recommended usage\") # 1) Configure the browser browser_config = BrowserConfig( headless=True, verbose=True ) # 2) Configure the crawler run crawler_run_config = CrawlerRunConfig( js_code=\"window.scrollTo(0, document.body.scrollHeight);\", wait_for=\"body\", cache_mode=CacheMode.BYPASS ) # 3) Create the crawler instance crawler = AsyncWebCrawler(config=browser_config) # # Define Hook Functions # async def on_browser_created(browser, **kwargs): # Called once the browser instance is created (but no pages or contexts yet) print(\"[HOOK] on_browser_created - Browser created successfully!\") # Typically, do minimal setup here if needed return browser async def on_page_context_created(page: Page, context: BrowserContext, **kwargs): # Called right after a new page + context are created (ideal for auth or route config). print(\"[HOOK] on_page_context_created - Setting up page & context.\") # Example 1: Route filtering (e.g., block images) async def route_filter(route): if route.request.resource_type == \"image\": print(f\"[HOOK] Blocking image request: {route.request.url}\") await route.abort() else: await route.continue_() await context.route(\"**\", route_filter) # Example 2: (Optional) Simulate a login scenario # (We do NOT create or close pages here, just do quick steps if needed) # e.g., await page.goto(\"https://example.com/login\") # e.g., await page.fill(\"input[name='username']\", \"testuser\") # e.g., await page.fill(\"input[name='password']\", \"password123\") # e.g., await page.click(\"button[type='submit']\") # e.g., await page.wait_for_selector(\"#welcome\") # e.g., await context.add_cookies([...]) # Then continue # Example 3: Adjust the viewport await page.set_viewport_size({\"width\": 1080, \"height\": 600}) return page async def before_goto( page: Page, context: BrowserContext, url: str, **kwargs ): # Called before navigating to each URL. print(f\"[HOOK] before_goto - About to navigate: {url}\") # e.g., inject custom headers await page.set_extra_http_headers({ \"Custom-Header\": \"my-value\" }) return page async def after_goto( page: Page, context: BrowserContext, url: str, response, **kwargs ): # Called after navigation completes. print(f\"[HOOK] after_goto - Successfully loaded: {url}\") # e.g., wait for a certain element if we want to verify try: await page.wait_for_selector('.content', timeout=1000) print(\"[HOOK] Found .content element!\") except: print(\"[HOOK] .content not found, continuing anyway.\") return page async def on_user_agent_updated( page: Page, context: BrowserContext, user_agent: str, **kwargs ): # Called whenever the user agent updates. print(f\"[HOOK] on_user_agent_updated - New user agent: {user_agent}\") return page async def on_execution_started(page: Page, context: BrowserContext, **kwargs): # Called after custom JavaScript execution begins. print(\"[HOOK] on_execution_started - JS code is running!\") return page async def before_retrieve_html(page: Page, context: BrowserContext, **kwargs): # Called before final HTML retrieval. print(\"[HOOK] before_retrieve_html - We can do final actions\") # Example: Scroll again await page.evaluate(\"window.scrollTo(0, document.body.scrollHeight);\") return page async def before_return_html( page: Page, context: BrowserContext, html: str, **kwargs ): # Called just before returning the HTML in the result. print(f\"[HOOK] before_return_html - HTML length: {len(html)}\") return page # # Attach Hooks # crawler.crawler_strategy.set_hook(\"on_browser_created\", on_browser_created) crawler.crawler_strategy.set_hook( \"on_page_context_created\", on_page_context_created ) crawler.crawler_strategy.set_hook(\"before_goto\", before_goto) crawler.crawler_strategy.set_hook(\"after_goto\", after_goto) crawler.crawler_strategy.set_hook( \"on_user_agent_updated\", on_user_agent_updated ) crawler.crawler_strategy.set_hook( \"on_execution_started\", on_execution_started ) crawler.crawler_strategy.set_hook( \"before_retrieve_html\", before_retrieve_html ) crawler.crawler_strategy.set_hook( \"before_return_html\", before_return_html ) await crawler.start() # 4) Run the crawler on an example page url = \"https://example.com\" result = await crawler.arun(url, config=crawler_run_config) if result.success: print(\"\\nCrawled URL:\", result.url) print(\"HTML length:\", len(result.html)) else: print(\"Error:\", result.error_message) await crawler.close() if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n## Hook Lifecycle Summary\n\n1. **`on_browser_created`** : - Browser is up, but **no** pages or contexts yet. - Light setup only—don’t try to open or close pages here (that belongs in `on_page_context_created`).\n\n2. **`on_page_context_created`** : - Perfect for advanced **auth** or route blocking. - You have a **page** + **context** ready but haven’t navigated to the target URL yet.\n\n3. **`before_goto`** : - Right before navigation. Typically used for setting **custom headers** or logging the target URL.\n\n4. **`after_goto`** : - After page navigation is done. Good place for verifying content or waiting on essential elements. \n\n5. **`on_user_agent_updated`** : - Whenever the user agent changes (for stealth or different UA modes).\n\n6. **`on_execution_started`** : - If you set `js_code` or run custom scripts, this runs once your JS is about to start.\n\n7. **`before_retrieve_html`** : - Just before the final HTML snapshot is taken. Often you do a final scroll or lazy-load triggers here.\n\n8. **`before_return_html`** : - The last hook before returning HTML to the `CrawlResult`. Good for logging HTML length or minor modifications.\n\n## When to Handle Authentication\n\n**Recommended** : Use **`on_page_context_created`** if you need to:\n\n * Navigate to a login page or fill forms\n * Set cookies or localStorage tokens\n * Block resource routes to avoid ads\n\n\n\nThis ensures the newly created context is under your control **before** `arun()` navigates to the main URL.\n\n## Additional Considerations\n\n * **Session Management** : If you want multiple `arun()` calls to reuse a single session, pass `session_id=` in your `CrawlerRunConfig`. Hooks remain the same. \n * **Performance** : Hooks can slow down crawling if they do heavy tasks. Keep them concise. \n * **Error Handling** : If a hook fails, the overall crawl might fail. Catch exceptions or handle them gracefully. \n * **Concurrency** : If you run `arun_many()`, each URL triggers these hooks in parallel. Ensure your hooks are thread/async-safe.\n\n\n\n## Conclusion\n\nHooks provide **fine-grained** control over:\n\n * **Browser** creation (light tasks only)\n * **Page** and **context** creation (auth, route blocking)\n * **Navigation** phases\n * **Final HTML** retrieval\n\n\n\nFollow the recommended usage: - **Login** or advanced tasks in `on_page_context_created` - **Custom headers** or logs in `before_goto` / `after_goto` - **Scrolling** or final checks in `before_retrieve_html` / `before_return_html`\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * Identity Based Crawling\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Preserve Your Identity with Crawl4AI](#preserve-your-identity-with-crawl4ai)\n * [1. Managed Browsers: Your Digital Identity Solution](#1-managed-browsers-your-digital-identity-solution)\n * [3. Using Managed Browsers in Crawl4AI](#3-using-managed-browsers-in-crawl4ai)\n * [4. Magic Mode: Simplified Automation](#4-magic-mode-simplified-automation)\n * [5. Comparing Managed Browsers vs. Magic Mode](#5-comparing-managed-browsers-vs-magic-mode)\n * [6. Using the BrowserProfiler Class](#6-using-the-browserprofiler-class)\n * [7. Summary](#7-summary)\n\n\n\n# Preserve Your Identity with Crawl4AI\n\nCrawl4AI empowers you to navigate and interact with the web using your **authentic digital identity** , ensuring you’re recognized as a human and not mistaken for a bot. This tutorial covers:\n\n1. **Managed Browsers** – The recommended approach for persistent profiles and identity-based crawling. 2. **Magic Mode** – A simplified fallback solution for quick automation without persistent identity.\n\n## 1. Managed Browsers: Your Digital Identity Solution\n\n**Managed Browsers** let developers create and use **persistent browser profiles**. These profiles store local storage, cookies, and other session data, letting you browse as your **real self** —complete with logins, preferences, and cookies.\n\n### Key Benefits\n\n * **Authentic Browsing Experience** : Retain session data and browser fingerprints as though you’re a normal user. \n * **Effortless Configuration** : Once you log in or solve CAPTCHAs in your chosen data directory, you can re-run crawls without repeating those steps. \n * **Empowered Data Access** : If you can see the data in your own browser, you can automate its retrieval with your genuine identity.\n\n\n\nBelow is a **partial update** to your **Managed Browsers** tutorial, specifically the section about **creating a user-data directory** using **Playwright’s Chromium** binary rather than a system-wide Chrome/Edge. We’ll show how to **locate** that binary and launch it with a `--user-data-dir` argument to set up your profile. You can then point `BrowserConfig.user_data_dir` to that folder for subsequent crawls.\n\n### Creating a User Data Directory (Command-Line Approach via Playwright)\n\nIf you installed Crawl4AI (which installs Playwright under the hood), you already have a Playwright-managed Chromium on your system. Follow these steps to launch that **Chromium** from your command line, specifying a **custom** data directory:\n\n1. **Find** the Playwright Chromium binary: - On most systems, installed browsers go under a `~/.cache/ms-playwright/` folder or similar path. - To see an overview of installed browsers, run: \n\n```\n`python -m playwright install --dry-run `\n```\n\nor \n\n```\n`playwright install --dry-run `\n```\n\n(depending on your environment). This shows where Playwright keeps Chromium. \n\n * For instance, you might see a path like: \n\n```\n`~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome `\n```\n\non Linux, or a corresponding folder on macOS/Windows.\n\n\n\n2. **Launch** the Playwright Chromium binary with a **custom** user-data directory: \n\n```\n`# Linux example ~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome \\ --user-data-dir=/home//my_chrome_profile `\n```\n\n```\n`# macOS example (Playwright’s internal binary) ~/Library/Caches/ms-playwright/chromium-1234/chrome-mac/Chromium.app/Contents/MacOS/Chromium \\ --user-data-dir=/Users//my_chrome_profile `\n```\n\n```\n`# Windows example (PowerShell/cmd) \"C:\\Users\\\\AppData\\Local\\ms-playwright\\chromium-1234\\chrome-win\\chrome.exe\" ^ --user-data-dir=\"C:\\Users\\\\my_chrome_profile\" `\n```\n\n**Replace** the path with the actual subfolder indicated in your `ms-playwright` cache structure. - This **opens** a fresh Chromium with your new or existing data folder. - **Log into** any sites or configure your browser the way you want. - **Close** when done—your profile data is saved in that folder.\n\n3. **Use** that folder in **`BrowserConfig.user_data_dir`** : \n\n```\n`from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig browser_config = BrowserConfig( headless=True, use_managed_browser=True, user_data_dir=\"/home//my_chrome_profile\", browser_type=\"chromium\" ) `\n```\n\n- Next time you run your code, it reuses that folder—**preserving** your session data, cookies, local storage, etc. \n\n## 3. Using Managed Browsers in Crawl4AI\n\nOnce you have a data directory with your session data, pass it to **`BrowserConfig`** :\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig async def main(): # 1) Reference your persistent data directory browser_config = BrowserConfig( headless=True, # 'True' for automated runs verbose=True, use_managed_browser=True, # Enables persistent browser strategy browser_type=\"chromium\", user_data_dir=\"/path/to/my-chrome-profile\" ) # 2) Standard crawl config crawl_config = CrawlerRunConfig( wait_for=\"css:.logged-in-content\" ) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url=\"https://example.com/private\", config=crawl_config) if result.success: print(\"Successfully accessed private data with your identity!\") else: print(\"Error:\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n### Workflow\n\n1. **Login** externally (via CLI or your normal Chrome with `--user-data-dir=...`). 2. **Close** that browser. 3. **Use** the same folder in `user_data_dir=` in Crawl4AI. 4. **Crawl** – The site sees your identity as if you’re the same user who just logged in.\n\n## 4. Magic Mode: Simplified Automation\n\nIf you **don’t** need a persistent profile or identity-based approach, **Magic Mode** offers a quick way to simulate human-like browsing without storing long-term data.\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://example.com\", config=CrawlerRunConfig( magic=True, # Simplifies a lot of interaction remove_overlay_elements=True, page_timeout=60000 ) ) `\n```\n\n**Magic Mode** :\n\n * Simulates a user-like experience \n * Randomizes user agent & navigator\n * Randomizes interactions & timings \n * Masks automation signals \n * Attempts pop-up handling \n\n\n\n**But** it’s no substitute for **true** user-based sessions if you want a fully legitimate identity-based solution.\n\n## 5. Comparing Managed Browsers vs. Magic Mode\n\nFeature | **Managed Browsers** | **Magic Mode** \n---|---|--- \n**Session Persistence** | Full localStorage/cookies retained in user_data_dir | No persistent data (fresh each run) \n**Genuine Identity** | Real user profile with full rights & preferences | Emulated user-like patterns, but no actual identity \n**Complex Sites** | Best for login-gated sites or heavy config | Simple tasks, minimal login or config needed \n**Setup** | External creation of user_data_dir, then use in Crawl4AI | Single-line approach (`magic=True`) \n**Reliability** | Extremely consistent (same data across runs) | Good for smaller tasks, can be less stable \n \n## 6. Using the BrowserProfiler Class\n\nCrawl4AI provides a dedicated `BrowserProfiler` class for managing browser profiles, making it easy to create, list, and delete profiles for identity-based browsing.\n\n### Creating and Managing Profiles with BrowserProfiler\n\nThe `BrowserProfiler` class offers a comprehensive API for browser profile management:\n\n```\n`import asyncio from crawl4ai import BrowserProfiler async def manage_profiles(): # Create a profiler instance profiler = BrowserProfiler() # Create a profile interactively - opens a browser window profile_path = await profiler.create_profile( profile_name=\"my-login-profile\" # Optional: name your profile ) print(f\"Profile saved at: {profile_path}\") # List all available profiles profiles = profiler.list_profiles() for profile in profiles: print(f\"Profile: {profile['name']}\") print(f\" Path: {profile['path']}\") print(f\" Created: {profile['created']}\") print(f\" Browser type: {profile['type']}\") # Get a specific profile path by name specific_profile = profiler.get_profile_path(\"my-login-profile\") # Delete a profile when no longer needed success = profiler.delete_profile(\"old-profile-name\") asyncio.run(manage_profiles()) `\n```\n\n**How profile creation works:** 1. A browser window opens for you to interact with 2. You log in to websites, set preferences, etc. 3. When you're done, press 'q' in the terminal to close the browser 4. The profile is saved in the Crawl4AI profiles directory 5. You can use the returned path with `BrowserConfig.user_data_dir`\n\n### Interactive Profile Management\n\nThe `BrowserProfiler` also offers an interactive management console that guides you through profile creation, listing, and deletion:\n\n```\n`import asyncio from crawl4ai import BrowserProfiler, AsyncWebCrawler, BrowserConfig # Define a function to use a profile for crawling async def crawl_with_profile(profile_path, url): browser_config = BrowserConfig( headless=True, use_managed_browser=True, user_data_dir=profile_path ) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url) return result async def main(): # Create a profiler instance profiler = BrowserProfiler() # Launch the interactive profile manager # Passing the crawl function as a callback adds a \"crawl with profile\" option await profiler.interactive_manager(crawl_callback=crawl_with_profile) asyncio.run(main()) `\n```\n\n### Legacy Methods\n\nFor backward compatibility, the previous methods on `ManagedBrowser` are still available, but they delegate to the new `BrowserProfiler` class:\n\n```\n`from crawl4ai.browser_manager import ManagedBrowser # These methods still work but use BrowserProfiler internally profiles = ManagedBrowser.list_profiles() `\n```\n\n### Complete Example\n\nSee the full example in `docs/examples/identity_based_browsing.py` for a complete demonstration of creating and using profiles for authenticated browsing using the new `BrowserProfiler` class.\n\n## 7. Summary\n\n * **Create** your user-data directory either:\n * By launching Chrome/Chromium externally with `--user-data-dir=/some/path`\n * Or by using the built-in `BrowserProfiler.create_profile()` method\n * Or through the interactive interface with `profiler.interactive_manager()`\n * **Log in** or configure sites as needed, then close the browser\n * **Reference** that folder in `BrowserConfig(user_data_dir=\"...\")` + `use_managed_browser=True`\n * **List and reuse** profiles with `BrowserProfiler.list_profiles()`\n * **Manage** your profiles with the dedicated `BrowserProfiler` class\n * Enjoy **persistent** sessions that reflect your real identity\n * If you only need quick, ephemeral automation, **Magic Mode** might suffice\n\n\n\n**Recommended** : Always prefer a **Managed Browser** for robust, identity-based crawling and simpler interactions with complex sites. Use **Magic Mode** for quick tasks or prototypes where persistent data is unnecessary.\n\nWith these approaches, you preserve your **authentic** browsing environment, ensuring the site sees you exactly as a normal user—no repeated logins or wasted time.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * Lazy Loading\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Handling Lazy-Loaded Images](#handling-lazy-loaded-images)\n * [Example: Ensuring Lazy Images Appear](#example-ensuring-lazy-images-appear)\n * [Combining with Other Link & Media Filters](#combining-with-other-link-media-filters)\n * [Tips & Troubleshooting](#tips-troubleshooting)\n\n\n\n## Handling Lazy-Loaded Images\n\nMany websites now load images **lazily** as you scroll. If you need to ensure they appear in your final crawl (and in `result.media`), consider:\n\n1. **`wait_for_images=True`** – Wait for images to fully load. 2. **`scan_full_page`** – Force the crawler to scroll the entire page, triggering lazy loads. 3. **`scroll_delay`** – Add small delays between scroll steps. \n\n**Note** : If the site requires multiple “Load More” triggers or complex interactions, see the [Page Interaction docs](../../core/page-interaction/).\n\n### Example: Ensuring Lazy Images Appear\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig from crawl4ai.async_configs import CacheMode async def main(): config = CrawlerRunConfig( # Force the crawler to wait until images are fully loaded wait_for_images=True, # Option 1: If you want to automatically scroll the page to load images scan_full_page=True, # Tells the crawler to try scrolling the entire page scroll_delay=0.5, # Delay (seconds) between scroll steps # Option 2: If the site uses a 'Load More' or JS triggers for images, # you can also specify js_code or wait_for logic here. cache_mode=CacheMode.BYPASS, verbose=True ) async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler: result = await crawler.arun(\"https://www.example.com/gallery\", config=config) if result.success: images = result.media.get(\"images\", []) print(\"Images found:\", len(images)) for i, img in enumerate(images[:5]): print(f\"[Image {i}] URL: {img['src']}, Score: {img.get('score','N/A')}\") else: print(\"Error:\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Explanation** :\n\n * **`wait_for_images=True`** The crawler tries to ensure images have finished loading before finalizing the HTML. \n * **`scan_full_page=True`** Tells the crawler to attempt scrolling from top to bottom. Each scroll step helps trigger lazy loading. \n * **`scroll_delay=0.5`** Pause half a second between each scroll step. Helps the site load images before continuing.\n\n\n\n**When to Use** :\n\n * **Lazy-Loading** : If images appear only when the user scrolls into view, `scan_full_page` + `scroll_delay` helps the crawler see them. \n * **Heavier Pages** : If a page is extremely long, be mindful that scanning the entire page can be slow. Adjust `scroll_delay` or the max scroll steps as needed.\n\n\n\n## Combining with Other Link & Media Filters\n\nYou can still combine **lazy-load** logic with the usual **exclude_external_images** , **exclude_domains** , or link filtration:\n\n```\n`config = CrawlerRunConfig( wait_for_images=True, scan_full_page=True, scroll_delay=0.5, # Filter out external images if you only want local ones exclude_external_images=True, # Exclude certain domains for links exclude_domains=[\"spammycdn.com\"], ) `\n```\n\nThis approach ensures you see **all** images from the main domain while ignoring external ones, and the crawler physically scrolls the entire page so that lazy-loading triggers.\n\n## Tips & Troubleshooting\n\n1. **Long Pages** - Setting `scan_full_page=True` on extremely long or infinite-scroll pages can be resource-intensive. - Consider using [hooks](../../core/page-interaction/) or specialized logic to load specific sections or “Load More” triggers repeatedly.\n\n2. **Mixed Image Behavior** - Some sites load images in batches as you scroll. If you’re missing images, increase your `scroll_delay` or call multiple partial scrolls in a loop with JS code or hooks.\n\n3. **Combining with Dynamic Wait** - If the site has a placeholder that only changes to a real image after a certain event, you might do `wait_for=\"css:img.loaded\"` or a custom JS `wait_for`.\n\n4. **Caching** - If `cache_mode` is enabled, repeated crawls might skip some network fetches. If you suspect caching is missing new images, set `cache_mode=CacheMode.BYPASS` for fresh fetches.\n\nWith **lazy-loading** support, **wait_for_images** , and **scan_full_page** settings, you can capture the entire gallery or feed of images you expect—even if the site only loads them as the user scrolls. Combine these with the standard media filtering and domain exclusion for a complete link & media handling strategy.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * Multi-URL Crawling\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Advanced Multi-URL Crawling with Dispatchers](#advanced-multi-url-crawling-with-dispatchers)\n * [1. Introduction](#1-introduction)\n * [2. Core Components](#2-core-components)\n * [3. Available Dispatchers](#3-available-dispatchers)\n * [4. Usage Examples](#4-usage-examples)\n * [5. Dispatch Results](#5-dispatch-results)\n * [6. Summary](#6-summary)\n\n\n\n# Advanced Multi-URL Crawling with Dispatchers\n\n> **Heads Up** : Crawl4AI supports advanced dispatchers for **parallel** or **throttled** crawling, providing dynamic rate limiting and memory usage checks. The built-in `arun_many()` function uses these dispatchers to handle concurrency efficiently.\n\n## 1. Introduction\n\nWhen crawling many URLs:\n\n * **Basic** : Use `arun()` in a loop (simple but less efficient)\n * **Better** : Use `arun_many()`, which efficiently handles multiple URLs with proper concurrency control\n * **Best** : Customize dispatcher behavior for your specific needs (memory management, rate limits, etc.)\n\n\n\n**Why Dispatchers?**\n\n * **Adaptive** : Memory-based dispatchers can pause or slow down based on system resources\n * **Rate-limiting** : Built-in rate limiting with exponential backoff for 429/503 responses\n * **Real-time Monitoring** : Live dashboard of ongoing tasks, memory usage, and performance\n * **Flexibility** : Choose between memory-adaptive or semaphore-based concurrency\n\n\n\n## 2. Core Components\n\n### 2.1 Rate Limiter\n\n```\n`class RateLimiter: def __init__( # Random delay range between requests base_delay: Tuple[float, float] = (1.0, 3.0), # Maximum backoff delay max_delay: float = 60.0, # Retries before giving up max_retries: int = 3, # Status codes triggering backoff rate_limit_codes: List[int] = [429, 503] ) `\n```\n\nHere’s the revised and simplified explanation of the **RateLimiter** , focusing on constructor parameters and adhering to your markdown style and mkDocs guidelines.\n\n#### RateLimiter Constructor Parameters\n\nThe **RateLimiter** is a utility that helps manage the pace of requests to avoid overloading servers or getting blocked due to rate limits. It operates internally to delay requests and handle retries but can be configured using its constructor parameters.\n\n**Parameters of the`RateLimiter` constructor:**\n\n1. **`base_delay`** (`Tuple[float, float]`, default: `(1.0, 3.0)`) The range for a random delay (in seconds) between consecutive requests to the same domain.\n\n * A random delay is chosen between `base_delay[0]` and `base_delay[1]` for each request. \n * This prevents sending requests at a predictable frequency, reducing the chances of triggering rate limits.\n\n\n\n**Example:** If `base_delay = (2.0, 5.0)`, delays could be randomly chosen as `2.3s`, `4.1s`, etc.\n\n2. **`max_delay`** (`float`, default: `60.0`) The maximum allowable delay when rate-limiting errors occur.\n\n * When servers return rate-limit responses (e.g., 429 or 503), the delay increases exponentially with jitter. \n * The `max_delay` ensures the delay doesn’t grow unreasonably high, capping it at this value.\n\n\n\n**Example:** For a `max_delay = 30.0`, even if backoff calculations suggest a delay of `45s`, it will cap at `30s`.\n\n3. **`max_retries`** (`int`, default: `3`) The maximum number of retries for a request if rate-limiting errors occur.\n\n * After encountering a rate-limit response, the `RateLimiter` retries the request up to this number of times. \n * If all retries fail, the request is marked as failed, and the process continues.\n\n\n\n**Example:** If `max_retries = 3`, the system retries a failed request three times before giving up.\n\n4. **`rate_limit_codes`** (`List[int]`, default: `[429, 503]`) A list of HTTP status codes that trigger the rate-limiting logic.\n\n * These status codes indicate the server is overwhelmed or actively limiting requests. \n * You can customize this list to include other codes based on specific server behavior.\n\n\n\n**Example:** If `rate_limit_codes = [429, 503, 504]`, the crawler will back off on these three error codes.\n\n**How to Use the`RateLimiter`:**\n\nHere’s an example of initializing and using a `RateLimiter` in your project:\n\n```\n`from crawl4ai import RateLimiter # Create a RateLimiter with custom settings rate_limiter = RateLimiter( base_delay=(2.0, 4.0), # Random delay between 2-4 seconds max_delay=30.0, # Cap delay at 30 seconds max_retries=5, # Retry up to 5 times on rate-limiting errors rate_limit_codes=[429, 503] # Handle these HTTP status codes ) # RateLimiter will handle delays and retries internally # No additional setup is required for its operation `\n```\n\nThe `RateLimiter` integrates seamlessly with dispatchers like `MemoryAdaptiveDispatcher` and `SemaphoreDispatcher`, ensuring requests are paced correctly without user intervention. Its internal mechanisms manage delays and retries to avoid overwhelming servers while maximizing efficiency.\n\n### 2.2 Crawler Monitor\n\nThe CrawlerMonitor provides real-time visibility into crawling operations:\n\n```\n`from crawl4ai import CrawlerMonitor, DisplayMode monitor = CrawlerMonitor( # Maximum rows in live display max_visible_rows=15, # DETAILED or AGGREGATED view display_mode=DisplayMode.DETAILED ) `\n```\n\n**Display Modes** :\n\n 1. **DETAILED** : Shows individual task status, memory usage, and timing\n 2. **AGGREGATED** : Displays summary statistics and overall progress\n\n\n\n## 3. Available Dispatchers\n\n### 3.1 MemoryAdaptiveDispatcher (Default)\n\nAutomatically manages concurrency based on system memory usage:\n\n```\n`from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=90.0, # Pause if memory exceeds this check_interval=1.0, # How often to check memory max_session_permit=10, # Maximum concurrent tasks rate_limiter=RateLimiter( # Optional rate limiting base_delay=(1.0, 2.0), max_delay=30.0, max_retries=2 ), monitor=CrawlerMonitor( # Optional monitoring max_visible_rows=15, display_mode=DisplayMode.DETAILED ) ) `\n```\n\n**Constructor Parameters:**\n\n1. **`memory_threshold_percent`** (`float`, default: `90.0`) Specifies the memory usage threshold (as a percentage). If system memory usage exceeds this value, the dispatcher pauses crawling to prevent system overload.\n\n2. **`check_interval`** (`float`, default: `1.0`) The interval (in seconds) at which the dispatcher checks system memory usage.\n\n3. **`max_session_permit`** (`int`, default: `10`) The maximum number of concurrent crawling tasks allowed. This ensures resource limits are respected while maintaining concurrency.\n\n4. **`memory_wait_timeout`** (`float`, default: `300.0`) Optional timeout (in seconds). If memory usage exceeds `memory_threshold_percent` for longer than this duration, a `MemoryError` is raised.\n\n5. **`rate_limiter`** (`RateLimiter`, default: `None`) Optional rate-limiting logic to avoid server-side blocking (e.g., for handling 429 or 503 errors). See **RateLimiter** for details.\n\n6. **`monitor`** (`CrawlerMonitor`, default: `None`) Optional monitoring for real-time task tracking and performance insights. See **CrawlerMonitor** for details.\n\n### 3.2 SemaphoreDispatcher\n\nProvides simple concurrency control with a fixed limit:\n\n```\n`from crawl4ai.async_dispatcher import SemaphoreDispatcher dispatcher = SemaphoreDispatcher( max_session_permit=20, # Maximum concurrent tasks rate_limiter=RateLimiter( # Optional rate limiting base_delay=(0.5, 1.0), max_delay=10.0 ), monitor=CrawlerMonitor( # Optional monitoring max_visible_rows=15, display_mode=DisplayMode.DETAILED ) ) `\n```\n\n**Constructor Parameters:**\n\n1. **`max_session_permit`** (`int`, default: `20`) The maximum number of concurrent crawling tasks allowed, irrespective of semaphore slots.\n\n2. **`rate_limiter`** (`RateLimiter`, default: `None`) Optional rate-limiting logic to avoid overwhelming servers. See **RateLimiter** for details.\n\n3. **`monitor`** (`CrawlerMonitor`, default: `None`) Optional monitoring for tracking task progress and resource usage. See **CrawlerMonitor** for details.\n\n## 4. Usage Examples\n\n### 4.1 Batch Processing (Default)\n\n```\n`async def crawl_batch(): browser_config = BrowserConfig(headless=True, verbose=False) run_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, stream=False # Default: get all results at once ) dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=70.0, check_interval=1.0, max_session_permit=10, monitor=CrawlerMonitor( display_mode=DisplayMode.DETAILED ) ) async with AsyncWebCrawler(config=browser_config) as crawler: # Get all results at once results = await crawler.arun_many( urls=urls, config=run_config, dispatcher=dispatcher ) # Process all results after completion for result in results: if result.success: await process_result(result) else: print(f\"Failed to crawl {result.url}: {result.error_message}\") `\n```\n\n**Review:** - **Purpose:** Executes a batch crawl with all URLs processed together after crawling is complete. - **Dispatcher:** Uses `MemoryAdaptiveDispatcher` to manage concurrency and system memory. - **Stream:** Disabled (`stream=False`), so all results are collected at once for post-processing. - **Best Use Case:** When you need to analyze results in bulk rather than individually during the crawl.\n\n### 4.2 Streaming Mode\n\n```\n`async def crawl_streaming(): browser_config = BrowserConfig(headless=True, verbose=False) run_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, stream=True # Enable streaming mode ) dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=70.0, check_interval=1.0, max_session_permit=10, monitor=CrawlerMonitor( display_mode=DisplayMode.DETAILED ) ) async with AsyncWebCrawler(config=browser_config) as crawler: # Process results as they become available async for result in await crawler.arun_many( urls=urls, config=run_config, dispatcher=dispatcher ): if result.success: # Process each result immediately await process_result(result) else: print(f\"Failed to crawl {result.url}: {result.error_message}\") `\n```\n\n**Review:** - **Purpose:** Enables streaming to process results as soon as they’re available. - **Dispatcher:** Uses `MemoryAdaptiveDispatcher` for concurrency and memory management. - **Stream:** Enabled (`stream=True`), allowing real-time processing during crawling. - **Best Use Case:** When you need to act on results immediately, such as for real-time analytics or progressive data storage.\n\n### 4.3 Semaphore-based Crawling\n\n```\n`async def crawl_with_semaphore(urls): browser_config = BrowserConfig(headless=True, verbose=False) run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) dispatcher = SemaphoreDispatcher( semaphore_count=5, rate_limiter=RateLimiter( base_delay=(0.5, 1.0), max_delay=10.0 ), monitor=CrawlerMonitor( max_visible_rows=15, display_mode=DisplayMode.DETAILED ) ) async with AsyncWebCrawler(config=browser_config) as crawler: results = await crawler.arun_many( urls, config=run_config, dispatcher=dispatcher ) return results `\n```\n\n**Review:** - **Purpose:** Uses `SemaphoreDispatcher` to limit concurrency with a fixed number of slots. - **Dispatcher:** Configured with a semaphore to control parallel crawling tasks. - **Rate Limiter:** Prevents servers from being overwhelmed by pacing requests. - **Best Use Case:** When you want precise control over the number of concurrent requests, independent of system memory.\n\n### 4.4 Robots.txt Consideration\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode async def main(): urls = [ \"https://example1.com\", \"https://example2.com\", \"https://example3.com\" ] config = CrawlerRunConfig( cache_mode=CacheMode.ENABLED, check_robots_txt=True, # Will respect robots.txt for each URL semaphore_count=3 # Max concurrent requests ) async with AsyncWebCrawler() as crawler: async for result in crawler.arun_many(urls, config=config): if result.success: print(f\"Successfully crawled {result.url}\") elif result.status_code == 403 and \"robots.txt\" in result.error_message: print(f\"Skipped {result.url} - blocked by robots.txt\") else: print(f\"Failed to crawl {result.url}: {result.error_message}\") if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Review:** - **Purpose:** Ensures compliance with `robots.txt` rules for ethical and legal web crawling. - **Configuration:** Set `check_robots_txt=True` to validate each URL against `robots.txt` before crawling. - **Dispatcher:** Handles requests with concurrency limits (`semaphore_count=3`). - **Best Use Case:** When crawling websites that strictly enforce robots.txt policies or for responsible crawling practices.\n\n## 5. Dispatch Results\n\nEach crawl result includes dispatch information:\n\n```\n`@dataclass class DispatchResult: task_id: str memory_usage: float peak_memory: float start_time: datetime end_time: datetime error_message: str = \"\" `\n```\n\nAccess via `result.dispatch_result`:\n\n```\n`for result in results: if result.success: dr = result.dispatch_result print(f\"URL: {result.url}\") print(f\"Memory: {dr.memory_usage:.1f}MB\") print(f\"Duration: {dr.end_time - dr.start_time}\") `\n```\n\n## 6. Summary\n\n1. **Two Dispatcher Types** :\n\n * MemoryAdaptiveDispatcher (default): Dynamic concurrency based on memory\n * SemaphoreDispatcher: Fixed concurrency limit\n\n\n\n2. **Optional Components** :\n\n * RateLimiter: Smart request pacing and backoff\n * CrawlerMonitor: Real-time progress visualization\n\n\n\n3. **Key Benefits** :\n\n * Automatic memory management\n * Built-in rate limiting\n * Live progress monitoring\n * Flexible concurrency control\n\n\n\nChoose the dispatcher that best fits your needs:\n\n * **MemoryAdaptiveDispatcher** : For large crawls or limited resources\n * **SemaphoreDispatcher** : For simple, fixed-concurrency scenarios\n\n\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * Proxy & Security\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Proxy](#proxy)\n * [Basic Proxy Setup](#basic-proxy-setup)\n * [Authenticated Proxy](#authenticated-proxy)\n * [Rotating Proxies](#rotating-proxies)\n\n\n\n# Proxy\n\n## Basic Proxy Setup\n\nSimple proxy configuration with `BrowserConfig`:\n\n```\n`from crawl4ai.async_configs import BrowserConfig # Using proxy URL browser_config = BrowserConfig(proxy=\"http://proxy.example.com:8080\") async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url=\"https://example.com\") # Using SOCKS proxy browser_config = BrowserConfig(proxy=\"socks5://proxy.example.com:1080\") async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url=\"https://example.com\") `\n```\n\n## Authenticated Proxy\n\nUse an authenticated proxy with `BrowserConfig`:\n\n```\n`from crawl4ai.async_configs import BrowserConfig proxy_config = { \"server\": \"http://proxy.example.com:8080\", \"username\": \"user\", \"password\": \"pass\" } browser_config = BrowserConfig(proxy_config=proxy_config) async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url=\"https://example.com\") `\n```\n\nHere's the corrected documentation:\n\n## Rotating Proxies\n\nExample using a proxy rotation service dynamically:\n\n```\n`from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig async def get_next_proxy(): # Your proxy rotation logic here return {\"server\": \"http://next.proxy.com:8080\"} async def main(): browser_config = BrowserConfig() run_config = CrawlerRunConfig() async with AsyncWebCrawler(config=browser_config) as crawler: # For each URL, create a new run config with different proxy for url in urls: proxy = await get_next_proxy() # Clone the config and update proxy - this creates a new browser context current_config = run_config.clone(proxy_config=proxy) result = await crawler.arun(url=url, config=current_config) if __name__ == \"__main__\": import asyncio asyncio.run(main()) `\n```\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * Session Management\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * [SSL Certificate](../ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Session Management](#session-management)\n * [Basic Session Usage](#basic-session-usage)\n * [Dynamic Content with Sessions](#dynamic-content-with-sessions)\n * [Example 1: Basic Session-Based Crawling](#example-1-basic-session-based-crawling)\n * [Advanced Technique 1: Custom Execution Hooks](#advanced-technique-1-custom-execution-hooks)\n * [Advanced Technique 2: Integrated JavaScript Execution and Waiting](#advanced-technique-2-integrated-javascript-execution-and-waiting)\n\n\n\n# Session Management\n\nSession management in Crawl4AI is a powerful feature that allows you to maintain state across multiple requests, making it particularly suitable for handling complex multi-step crawling tasks. It enables you to reuse the same browser tab (or page object) across sequential actions and crawls, which is beneficial for:\n\n * **Performing JavaScript actions before and after crawling.**\n * **Executing multiple sequential crawls faster** without needing to reopen tabs or allocate memory repeatedly.\n\n\n\n**Note:** This feature is designed for sequential workflows and is not suitable for parallel operations.\n\n#### Basic Session Usage\n\nUse `BrowserConfig` and `CrawlerRunConfig` to maintain state with a `session_id`:\n\n```\n`from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig async with AsyncWebCrawler() as crawler: session_id = \"my_session\" # Define configurations config1 = CrawlerRunConfig( url=\"https://example.com/page1\", session_id=session_id ) config2 = CrawlerRunConfig( url=\"https://example.com/page2\", session_id=session_id ) # First request result1 = await crawler.arun(config=config1) # Subsequent request using the same session result2 = await crawler.arun(config=config2) # Clean up when done await crawler.crawler_strategy.kill_session(session_id) `\n```\n\n#### Dynamic Content with Sessions\n\nHere's an example of crawling GitHub commits across multiple pages while preserving session state:\n\n```\n`from crawl4ai.async_configs import CrawlerRunConfig from crawl4ai.extraction_strategy import JsonCssExtractionStrategy from crawl4ai.cache_context import CacheMode async def crawl_dynamic_content(): async with AsyncWebCrawler() as crawler: session_id = \"github_commits_session\" url = \"https://github.com/microsoft/TypeScript/commits/main\" all_commits = [] # Define extraction schema schema = { \"name\": \"Commit Extractor\", \"baseSelector\": \"li.Box-sc-g0xbh4-0\", \"fields\": [{ \"name\": \"title\", \"selector\": \"h4.markdown-title\", \"type\": \"text\" }], } extraction_strategy = JsonCssExtractionStrategy(schema) # JavaScript and wait configurations js_next_page = \"\"\"document.querySelector('a[data-testid=\"pagination-next-button\"]').click();\"\"\" wait_for = \"\"\"() => document.querySelectorAll('li.Box-sc-g0xbh4-0').length > 0\"\"\" # Crawl multiple pages for page in range(3): config = CrawlerRunConfig( url=url, session_id=session_id, extraction_strategy=extraction_strategy, js_code=js_next_page if page > 0 else None, wait_for=wait_for if page > 0 else None, js_only=page > 0, cache_mode=CacheMode.BYPASS ) result = await crawler.arun(config=config) if result.success: commits = json.loads(result.extracted_content) all_commits.extend(commits) print(f\"Page {page + 1}: Found {len(commits)} commits\") # Clean up session await crawler.crawler_strategy.kill_session(session_id) return all_commits `\n```\n\n## Example 1: Basic Session-Based Crawling\n\nA simple example using session-based crawling:\n\n```\n`import asyncio from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig from crawl4ai.cache_context import CacheMode async def basic_session_crawl(): async with AsyncWebCrawler() as crawler: session_id = \"dynamic_content_session\" url = \"https://example.com/dynamic-content\" for page in range(3): config = CrawlerRunConfig( url=url, session_id=session_id, js_code=\"document.querySelector('.load-more-button').click();\" if page > 0 else None, css_selector=\".content-item\", cache_mode=CacheMode.BYPASS ) result = await crawler.arun(config=config) print(f\"Page {page + 1}: Found {result.extracted_content.count('.content-item')} items\") await crawler.crawler_strategy.kill_session(session_id) asyncio.run(basic_session_crawl()) `\n```\n\nThis example shows: 1. Reusing the same `session_id` across multiple requests. 2. Executing JavaScript to load more content dynamically. 3. Properly closing the session to free resources.\n\n## Advanced Technique 1: Custom Execution Hooks\n\n> Warning: You might feel confused by the end of the next few examples 😅, so make sure you are comfortable with the order of the parts before you start this.\n\nUse custom hooks to handle complex scenarios, such as waiting for content to load dynamically:\n\n```\n`async def advanced_session_crawl_with_hooks(): first_commit = \"\" async def on_execution_started(page): nonlocal first_commit try: while True: await page.wait_for_selector(\"li.commit-item h4\") commit = await page.query_selector(\"li.commit-item h4\") commit = await commit.evaluate(\"(element) => element.textContent\").strip() if commit and commit != first_commit: first_commit = commit break await asyncio.sleep(0.5) except Exception as e: print(f\"Warning: New content didn't appear: {e}\") async with AsyncWebCrawler() as crawler: session_id = \"commit_session\" url = \"https://github.com/example/repo/commits/main\" crawler.crawler_strategy.set_hook(\"on_execution_started\", on_execution_started) js_next_page = \"\"\"document.querySelector('a.pagination-next').click();\"\"\" for page in range(3): config = CrawlerRunConfig( url=url, session_id=session_id, js_code=js_next_page if page > 0 else None, css_selector=\"li.commit-item\", js_only=page > 0, cache_mode=CacheMode.BYPASS ) result = await crawler.arun(config=config) print(f\"Page {page + 1}: Found {len(result.extracted_content)} commits\") await crawler.crawler_strategy.kill_session(session_id) asyncio.run(advanced_session_crawl_with_hooks()) `\n```\n\nThis technique ensures new content loads before the next action.\n\n## Advanced Technique 2: Integrated JavaScript Execution and Waiting\n\nCombine JavaScript execution and waiting logic for concise handling of dynamic content:\n\n```\n`async def integrated_js_and_wait_crawl(): async with AsyncWebCrawler() as crawler: session_id = \"integrated_session\" url = \"https://github.com/example/repo/commits/main\" js_next_page_and_wait = \"\"\" (async () => { const getCurrentCommit = () => document.querySelector('li.commit-item h4').textContent.trim(); const initialCommit = getCurrentCommit(); document.querySelector('a.pagination-next').click(); while (getCurrentCommit() === initialCommit) { await new Promise(resolve => setTimeout(resolve, 100)); } })(); \"\"\" for page in range(3): config = CrawlerRunConfig( url=url, session_id=session_id, js_code=js_next_page_and_wait if page > 0 else None, css_selector=\"li.commit-item\", js_only=page > 0, cache_mode=CacheMode.BYPASS ) result = await crawler.arun(config=config) print(f\"Page {page + 1}: Found {len(result.extracted_content)} commits\") await crawler.crawler_strategy.kill_session(session_id) asyncio.run(integrated_js_and_wait_crawl()) `\n```\n\n#### Common Use Cases for Sessions\n\n1. **Authentication Flows** : Login and interact with secured pages.\n\n2. **Pagination Handling** : Navigate through multiple pages.\n\n3. **Form Submissions** : Fill forms, submit, and process results.\n\n4. **Multi-step Processes** : Complete workflows that span multiple actions.\n\n5. **Dynamic Content Navigation** : Handle JavaScript-rendered or event-triggered content.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../advanced-features/)\n * [File Downloading](../file-downloading/)\n * [Lazy Loading](../lazy-loading/)\n * [Hooks & Auth](../hooks-auth/)\n * [Proxy & Security](../proxy-security/)\n * [Session Management](../session-management/)\n * [Multi-URL Crawling](../multi-url-crawling/)\n * [Crawl Dispatcher](../crawl-dispatcher/)\n * [Identity Based Crawling](../identity-based-crawling/)\n * SSL Certificate\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [SSLCertificate Reference](#sslcertificate-reference)\n * [1. Overview](#1-overview)\n * [2. Construction & Fetching](#2-construction-fetching)\n * [3. Common Properties](#3-common-properties)\n * [4. Export Methods](#4-export-methods)\n * [5. Example Usage in Crawl4AI](#5-example-usage-in-crawl4ai)\n * [6. Notes & Best Practices](#6-notes-best-practices)\n\n\n\n# `SSLCertificate` Reference\n\nThe **`SSLCertificate`** class encapsulates an SSL certificate’s data and allows exporting it in various formats (PEM, DER, JSON, or text). It’s used within **Crawl4AI** whenever you set **`fetch_ssl_certificate=True`** in your **`CrawlerRunConfig`**. \n\n## 1. Overview\n\n**Location** : `crawl4ai/ssl_certificate.py`\n\n```\n`class SSLCertificate: \"\"\" Represents an SSL certificate with methods to export in various formats. Main Methods: - from_url(url, timeout=10) - from_file(file_path) - from_binary(binary_data) - to_json(filepath=None) - to_pem(filepath=None) - to_der(filepath=None) ... Common Properties: - issuer - subject - valid_from - valid_until - fingerprint \"\"\" `\n```\n\n### Typical Use Case\n\n 1. You **enable** certificate fetching in your crawl by: \n\n```\n`CrawlerRunConfig(fetch_ssl_certificate=True, ...) `\n```\n\n 2. After `arun()`, if `result.ssl_certificate` is present, it’s an instance of **`SSLCertificate`**. \n 3. You can **read** basic properties (issuer, subject, validity) or **export** them in multiple formats.\n\n\n\n## 2. Construction & Fetching\n\n### 2.1 **`from_url(url, timeout=10)`**\n\nManually load an SSL certificate from a given URL (port 443). Typically used internally, but you can call it directly if you want:\n\n```\n`cert = SSLCertificate.from_url(\"https://example.com\") if cert: print(\"Fingerprint:\", cert.fingerprint) `\n```\n\n### 2.2 **`from_file(file_path)`**\n\nLoad from a file containing certificate data in ASN.1 or DER. Rarely needed unless you have local cert files:\n\n```\n`cert = SSLCertificate.from_file(\"/path/to/cert.der\") `\n```\n\n### 2.3 **`from_binary(binary_data)`**\n\nInitialize from raw binary. E.g., if you captured it from a socket or another source:\n\n```\n`cert = SSLCertificate.from_binary(raw_bytes) `\n```\n\n## 3. Common Properties\n\nAfter obtaining a **`SSLCertificate`** instance (e.g. `result.ssl_certificate` from a crawl), you can read:\n\n1. **`issuer`** _(dict)_ - E.g. `{\"CN\": \"My Root CA\", \"O\": \"...\"}` 2. **`subject`** _(dict)_ - E.g. `{\"CN\": \"example.com\", \"O\": \"ExampleOrg\"}` 3. **`valid_from`** _(str)_ - NotBefore date/time. Often in ASN.1/UTC format. 4. **`valid_until`** _(str)_ - NotAfter date/time. 5. **`fingerprint`** _(str)_ - The SHA-256 digest (lowercase hex). - E.g. `\"d14d2e...\"`\n\n## 4. Export Methods\n\nOnce you have a **`SSLCertificate`** object, you can **export** or **inspect** it:\n\n### 4.1 **`to_json(filepath=None)` → `Optional[str]`**\n\n * Returns a JSON string containing the parsed certificate fields. \n * If `filepath` is provided, saves it to disk instead, returning `None`.\n\n\n\n**Usage** : \n\n```\n`json_data = cert.to_json() # returns JSON string cert.to_json(\"certificate.json\") # writes file, returns None `\n```\n\n### 4.2 **`to_pem(filepath=None)` → `Optional[str]`**\n\n * Returns a PEM-encoded string (common for web servers). \n * If `filepath` is provided, saves it to disk instead.\n\n\n\n```\n`pem_str = cert.to_pem() # in-memory PEM string cert.to_pem(\"/path/to/cert.pem\") # saved to file `\n```\n\n### 4.3 **`to_der(filepath=None)` → `Optional[bytes]`**\n\n * Returns the original DER (binary ASN.1) bytes. \n * If `filepath` is specified, writes the bytes there instead.\n\n\n\n```\n`der_bytes = cert.to_der() cert.to_der(\"certificate.der\") `\n```\n\n### 4.4 (Optional) **`export_as_text()`**\n\n * If you see a method like `export_as_text()`, it typically returns an OpenSSL-style textual representation. \n * Not always needed, but can help for debugging or manual inspection.\n\n\n\n## 5. Example Usage in Crawl4AI\n\nBelow is a minimal sample showing how the crawler obtains an SSL cert from a site, then reads or exports it. The code snippet:\n\n```\n`import asyncio import os from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode async def main(): tmp_dir = \"tmp\" os.makedirs(tmp_dir, exist_ok=True) config = CrawlerRunConfig( fetch_ssl_certificate=True, cache_mode=CacheMode.BYPASS ) async with AsyncWebCrawler() as crawler: result = await crawler.arun(\"https://example.com\", config=config) if result.success and result.ssl_certificate: cert = result.ssl_certificate # 1. Basic Info print(\"Issuer CN:\", cert.issuer.get(\"CN\", \"\")) print(\"Valid until:\", cert.valid_until) print(\"Fingerprint:\", cert.fingerprint) # 2. Export cert.to_json(os.path.join(tmp_dir, \"certificate.json\")) cert.to_pem(os.path.join(tmp_dir, \"certificate.pem\")) cert.to_der(os.path.join(tmp_dir, \"certificate.der\")) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n## 6. Notes & Best Practices\n\n1. **Timeout** : `SSLCertificate.from_url` internally uses a default **10s** socket connect and wraps SSL. 2. **Binary Form** : The certificate is loaded in ASN.1 (DER) form, then re-parsed by `OpenSSL.crypto`. 3. **Validation** : This does **not** validate the certificate chain or trust store. It only fetches and parses. 4. **Integration** : Within Crawl4AI, you typically just set `fetch_ssl_certificate=True` in `CrawlerRunConfig`; the final result’s `ssl_certificate` is automatically built. 5. **Export** : If you need to store or analyze a cert, the `to_json` and `to_pem` are quite universal.\n\n### Summary\n\n * **`SSLCertificate`** is a convenience class for capturing and exporting the **TLS certificate** from your crawled site(s). \n * Common usage is in the **`CrawlResult.ssl_certificate`** field, accessible after setting `fetch_ssl_certificate=True`. \n * Offers quick access to essential certificate details (`issuer`, `subject`, `fingerprint`) and is easy to export (PEM, DER, JSON) for further analysis or server usage.\n\n\n\nUse it whenever you need **insight** into a site’s certificate or require some form of cryptographic or compliance check.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../async-webcrawler/)\n * arun()\n * [arun_many()](../arun_many/)\n * [Browser, Crawler & LLM Config](../parameters/)\n * [CrawlResult](../crawl-result/)\n * [Strategies](../strategies/)\n\n\n\n * [arun() Parameter Guide (New Approach)](#arun-parameter-guide-new-approach)\n * [1. Core Usage](#1-core-usage)\n * [2. Cache Control](#2-cache-control)\n * [3. Content Processing & Selection](#3-content-processing-selection)\n * [4. Page Navigation & Timing](#4-page-navigation-timing)\n * [5. Session Management](#5-session-management)\n * [6. Screenshot, PDF & Media Options](#6-screenshot-pdf-media-options)\n * [7. Extraction Strategy](#7-extraction-strategy)\n * [8. Comprehensive Example](#8-comprehensive-example)\n * [9. Best Practices](#9-best-practices)\n * [10. Conclusion](#10-conclusion)\n\n\n\n# `arun()` Parameter Guide (New Approach)\n\nIn Crawl4AI’s **latest** configuration model, nearly all parameters that once went directly to `arun()` are now part of **`CrawlerRunConfig`**. When calling `arun()`, you provide:\n\n```\n`await crawler.arun( url=\"https://example.com\", config=my_run_config ) `\n```\n\nBelow is an organized look at the parameters that can go inside `CrawlerRunConfig`, divided by their functional areas. For **Browser** settings (e.g., `headless`, `browser_type`), see [BrowserConfig](../parameters/).\n\n## 1. Core Usage\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode async def main(): run_config = CrawlerRunConfig( verbose=True, # Detailed logging cache_mode=CacheMode.ENABLED, # Use normal read/write cache check_robots_txt=True, # Respect robots.txt rules # ... other parameters ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://example.com\", config=run_config ) # Check if blocked by robots.txt if not result.success and result.status_code == 403: print(f\"Error: {result.error_message}\") `\n```\n\n**Key Fields** : - `verbose=True` logs each crawl step. - `cache_mode` decides how to read/write the local crawl cache.\n\n## 2. Cache Control\n\n**`cache_mode`** (default: `CacheMode.ENABLED`) Use a built-in enum from `CacheMode`:\n\n * `ENABLED`: Normal caching—reads if available, writes if missing.\n * `DISABLED`: No caching—always refetch pages.\n * `READ_ONLY`: Reads from cache only; no new writes.\n * `WRITE_ONLY`: Writes to cache but doesn’t read existing data.\n * `BYPASS`: Skips reading cache for this crawl (though it might still write if set up that way).\n\n\n\n```\n`run_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS ) `\n```\n\n**Additional flags** :\n\n * `bypass_cache=True` acts like `CacheMode.BYPASS`.\n * `disable_cache=True` acts like `CacheMode.DISABLED`.\n * `no_cache_read=True` acts like `CacheMode.WRITE_ONLY`.\n * `no_cache_write=True` acts like `CacheMode.READ_ONLY`.\n\n\n\n## 3. Content Processing & Selection\n\n### 3.1 Text Processing\n\n```\n`run_config = CrawlerRunConfig( word_count_threshold=10, # Ignore text blocks <10 words only_text=False, # If True, tries to remove non-text elements keep_data_attributes=False # Keep or discard data-* attributes ) `\n```\n\n### 3.2 Content Selection\n\n```\n`run_config = CrawlerRunConfig( css_selector=\".main-content\", # Focus on .main-content region only excluded_tags=[\"form\", \"nav\"], # Remove entire tag blocks remove_forms=True, # Specifically strip
elements remove_overlay_elements=True, # Attempt to remove modals/popups ) `\n```\n\n### 3.3 Link Handling\n\n```\n`run_config = CrawlerRunConfig( exclude_external_links=True, # Remove external links from final content exclude_social_media_links=True, # Remove links to known social sites exclude_domains=[\"ads.example.com\"], # Exclude links to these domains exclude_social_media_domains=[\"facebook.com\",\"twitter.com\"], # Extend the default list ) `\n```\n\n### 3.4 Media Filtering\n\n```\n`run_config = CrawlerRunConfig( exclude_external_images=True # Strip images from other domains ) `\n```\n\n## 4. Page Navigation & Timing\n\n### 4.1 Basic Browser Flow\n\n```\n`run_config = CrawlerRunConfig( wait_for=\"css:.dynamic-content\", # Wait for .dynamic-content delay_before_return_html=2.0, # Wait 2s before capturing final HTML page_timeout=60000, # Navigation & script timeout (ms) ) `\n```\n\n**Key Fields** :\n\n * `wait_for`: \n * `\"css:selector\"` or \n * `\"js:() => boolean\"` e.g. `js:() => document.querySelectorAll('.item').length > 10`.\n\n * `mean_delay` & `max_range`: define random delays for `arun_many()` calls. \n\n * `semaphore_count`: concurrency limit when crawling multiple URLs.\n\n\n\n### 4.2 JavaScript Execution\n\n```\n`run_config = CrawlerRunConfig( js_code=[ \"window.scrollTo(0, document.body.scrollHeight);\", \"document.querySelector('.load-more')?.click();\" ], js_only=False ) `\n```\n\n * `js_code` can be a single string or a list of strings. \n * `js_only=True` means “I’m continuing in the same session with new JS steps, no new full navigation.”\n\n\n\n### 4.3 Anti-Bot\n\n```\n`run_config = CrawlerRunConfig( magic=True, simulate_user=True, override_navigator=True ) `\n```\n\n- `magic=True` tries multiple stealth features. - `simulate_user=True` mimics mouse movements or random delays. - `override_navigator=True` fakes some navigator properties (like user agent checks). \n\n## 5. Session Management\n\n**`session_id`** : \n\n```\n`run_config = CrawlerRunConfig( session_id=\"my_session123\" ) `\n```\n\nIf re-used in subsequent `arun()` calls, the same tab/page context is continued (helpful for multi-step tasks or stateful browsing). \n\n## 6. Screenshot, PDF & Media Options\n\n```\n`run_config = CrawlerRunConfig( screenshot=True, # Grab a screenshot as base64 screenshot_wait_for=1.0, # Wait 1s before capturing pdf=True, # Also produce a PDF image_description_min_word_threshold=5, # If analyzing alt text image_score_threshold=3, # Filter out low-score images ) `\n```\n\n**Where they appear** : - `result.screenshot` → Base64 screenshot string. - `result.pdf` → Byte array with PDF data. \n\n## 7. Extraction Strategy\n\n**For advanced data extraction** (CSS/LLM-based), set `extraction_strategy`:\n\n```\n`run_config = CrawlerRunConfig( extraction_strategy=my_css_or_llm_strategy ) `\n```\n\nThe extracted data will appear in `result.extracted_content`.\n\n## 8. Comprehensive Example\n\nBelow is a snippet combining many parameters:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode from crawl4ai.extraction_strategy import JsonCssExtractionStrategy async def main(): # Example schema schema = { \"name\": \"Articles\", \"baseSelector\": \"article.post\", \"fields\": [ {\"name\": \"title\", \"selector\": \"h2\", \"type\": \"text\"}, {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"} ] } run_config = CrawlerRunConfig( # Core verbose=True, cache_mode=CacheMode.ENABLED, check_robots_txt=True, # Respect robots.txt rules # Content word_count_threshold=10, css_selector=\"main.content\", excluded_tags=[\"nav\", \"footer\"], exclude_external_links=True, # Page & JS js_code=\"document.querySelector('.show-more')?.click();\", wait_for=\"css:.loaded-block\", page_timeout=30000, # Extraction extraction_strategy=JsonCssExtractionStrategy(schema), # Session session_id=\"persistent_session\", # Media screenshot=True, pdf=True, # Anti-bot simulate_user=True, magic=True, ) async with AsyncWebCrawler() as crawler: result = await crawler.arun(\"https://example.com/posts\", config=run_config) if result.success: print(\"HTML length:\", len(result.cleaned_html)) print(\"Extraction JSON:\", result.extracted_content) if result.screenshot: print(\"Screenshot length:\", len(result.screenshot)) if result.pdf: print(\"PDF bytes length:\", len(result.pdf)) else: print(\"Error:\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**What we covered** :\n\n1. **Crawling** the main content region, ignoring external links. 2. Running **JavaScript** to click “.show-more”. 3. **Waiting** for “.loaded-block” to appear. 4. Generating a **screenshot** & **PDF** of the final page. 5. Extracting repeated “article.post” elements with a **CSS-based** extraction strategy.\n\n## 9. Best Practices\n\n1. **Use`BrowserConfig` for global browser** settings (headless, user agent). 2. **Use`CrawlerRunConfig`** to handle the **specific** crawl needs: content filtering, caching, JS, screenshot, extraction, etc. 3. Keep your **parameters consistent** in run configs—especially if you’re part of a large codebase with multiple crawls. 4. **Limit** large concurrency (`semaphore_count`) if the site or your system can’t handle it. 5. For dynamic pages, set `js_code` or `scan_full_page` so you load all content.\n\n## 10. Conclusion\n\nAll parameters that used to be direct arguments to `arun()` now belong in **`CrawlerRunConfig`**. This approach:\n\n * Makes code **clearer** and **more maintainable**. \n * Minimizes confusion about which arguments affect global vs. per-crawl behavior. \n * Allows you to create **reusable** config objects for different pages or tasks.\n\n\n\nFor a **full** reference, check out the [CrawlerRunConfig Docs](../parameters/). \n\nHappy crawling with your **structured, flexible** config approach!\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../async-webcrawler/)\n * [arun()](../arun/)\n * arun_many()\n * [Browser, Crawler & LLM Config](../parameters/)\n * [CrawlResult](../crawl-result/)\n * [Strategies](../strategies/)\n\n\n\n * [arun_many(...) Reference](#arun_many-reference)\n * [Function Signature](#function-signature)\n * [Differences from arun()](#differences-from-arun)\n * [Dispatcher Reference](#dispatcher-reference)\n * [Common Pitfalls](#common-pitfalls)\n * [Conclusion](#conclusion)\n\n\n\n# `arun_many(...)` Reference\n\n> **Note** : This function is very similar to [`arun()`](../arun/) but focused on **concurrent** or **batch** crawling. If you’re unfamiliar with `arun()` usage, please read that doc first, then review this for differences.\n\n## Function Signature\n\n```\n`async def arun_many( urls: Union[List[str], List[Any]], config: Optional[CrawlerRunConfig] = None, dispatcher: Optional[BaseDispatcher] = None, ... ) -> Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]: \"\"\" Crawl multiple URLs concurrently or in batches. :param urls: A list of URLs (or tasks) to crawl. :param config: (Optional) A default `CrawlerRunConfig` applying to each crawl. :param dispatcher: (Optional) A concurrency controller (e.g. MemoryAdaptiveDispatcher). ... :return: Either a list of `CrawlResult` objects, or an async generator if streaming is enabled. \"\"\" `\n```\n\n## Differences from `arun()`\n\n1. **Multiple URLs** : \n\n * Instead of crawling a single URL, you pass a list of them (strings or tasks). \n * The function returns either a **list** of `CrawlResult` or an **async generator** if streaming is enabled.\n\n\n\n2. **Concurrency & Dispatchers**: \n\n * **`dispatcher`** param allows advanced concurrency control. \n * If omitted, a default dispatcher (like `MemoryAdaptiveDispatcher`) is used internally. \n * Dispatchers handle concurrency, rate limiting, and memory-based adaptive throttling (see [Multi-URL Crawling](../../advanced/multi-url-crawling/)).\n\n\n\n3. **Streaming Support** : \n\n * Enable streaming by setting `stream=True` in your `CrawlerRunConfig`.\n * When streaming, use `async for` to process results as they become available.\n * Ideal for processing large numbers of URLs without waiting for all to complete.\n\n\n\n4. **Parallel** Execution**: \n\n * `arun_many()` can run multiple requests concurrently under the hood. \n * Each `CrawlResult` might also include a **`dispatch_result`** with concurrency details (like memory usage, start/end times).\n\n\n\n### Basic Example (Batch Mode)\n\n```\n`# Minimal usage: The default dispatcher will be used results = await crawler.arun_many( urls=[\"https://site1.com\", \"https://site2.com\"], config=CrawlerRunConfig(stream=False) # Default behavior ) for res in results: if res.success: print(res.url, \"crawled OK!\") else: print(\"Failed:\", res.url, \"-\", res.error_message) `\n```\n\n### Streaming Example\n\n```\n`config = CrawlerRunConfig( stream=True, # Enable streaming mode cache_mode=CacheMode.BYPASS ) # Process results as they complete async for result in await crawler.arun_many( urls=[\"https://site1.com\", \"https://site2.com\", \"https://site3.com\"], config=config ): if result.success: print(f\"Just completed: {result.url}\") # Process each result immediately process_result(result) `\n```\n\n### With a Custom Dispatcher\n\n```\n`dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=70.0, max_session_permit=10 ) results = await crawler.arun_many( urls=[\"https://site1.com\", \"https://site2.com\", \"https://site3.com\"], config=my_run_config, dispatcher=dispatcher ) `\n```\n\n**Key Points** : - Each URL is processed by the same or separate sessions, depending on the dispatcher’s strategy. - `dispatch_result` in each `CrawlResult` (if using concurrency) can hold memory and timing info. - If you need to handle authentication or session IDs, pass them in each individual task or within your run config.\n\n### Return Value\n\nEither a **list** of [`CrawlResult`](../crawl-result/) objects, or an **async generator** if streaming is enabled. You can iterate to check `result.success` or read each item’s `extracted_content`, `markdown`, or `dispatch_result`.\n\n## Dispatcher Reference\n\n * **`MemoryAdaptiveDispatcher`** : Dynamically manages concurrency based on system memory usage. \n * **`SemaphoreDispatcher`** : Fixed concurrency limit, simpler but less adaptive. \n\n\n\nFor advanced usage or custom settings, see [Multi-URL Crawling with Dispatchers](../../advanced/multi-url-crawling/).\n\n## Common Pitfalls\n\n1. **Large Lists** : If you pass thousands of URLs, be mindful of memory or rate-limits. A dispatcher can help. \n\n2. **Session Reuse** : If you need specialized logins or persistent contexts, ensure your dispatcher or tasks handle sessions accordingly. \n\n3. **Error Handling** : Each `CrawlResult` might fail for different reasons—always check `result.success` or the `error_message` before proceeding.\n\n## Conclusion\n\nUse `arun_many()` when you want to **crawl multiple URLs** simultaneously or in controlled parallel tasks. If you need advanced concurrency features (like memory-based adaptive throttling or complex rate-limiting), provide a **dispatcher**. Each result is a standard `CrawlResult`, possibly augmented with concurrency stats (`dispatch_result`) for deeper inspection. For more details on concurrency logic and dispatchers, see the [Advanced Multi-URL Crawling](../../advanced/multi-url-crawling/) docs.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * AsyncWebCrawler\n * [arun()](../arun/)\n * [arun_many()](../arun_many/)\n * [Browser, Crawler & LLM Config](../parameters/)\n * [CrawlResult](../crawl-result/)\n * [Strategies](../strategies/)\n\n\n\n * [AsyncWebCrawler](#asyncwebcrawler)\n * [1. Constructor Overview](#1-constructor-overview)\n * [2. Lifecycle: Start/Close or Context Manager](#2-lifecycle-startclose-or-context-manager)\n * [3. Primary Method: arun()](#3-primary-method-arun)\n * [4. Batch Processing: arun_many()](#4-batch-processing-arun_many)\n * [7. Best Practices & Migration Notes](#7-best-practices-migration-notes)\n * [8. Summary](#8-summary)\n\n\n\n# AsyncWebCrawler\n\nThe **`AsyncWebCrawler`** is the core class for asynchronous web crawling in Crawl4AI. You typically create it **once** , optionally customize it with a **`BrowserConfig`** (e.g., headless, user agent), then **run** multiple **`arun()`** calls with different **`CrawlerRunConfig`** objects.\n\n**Recommended usage** :\n\n1. **Create** a `BrowserConfig` for global browser settings. \n\n2. **Instantiate** `AsyncWebCrawler(config=browser_config)`. \n\n3. **Use** the crawler in an async context manager (`async with`) or manage start/close manually. \n\n4. **Call** `arun(url, config=crawler_run_config)` for each page you want.\n\n## 1. Constructor Overview\n\n```\n`class AsyncWebCrawler: def __init__( self, crawler_strategy: Optional[AsyncCrawlerStrategy] = None, config: Optional[BrowserConfig] = None, always_bypass_cache: bool = False, # deprecated always_by_pass_cache: Optional[bool] = None, # also deprecated base_directory: str = ..., thread_safe: bool = False, **kwargs, ): \"\"\" Create an AsyncWebCrawler instance. Args: crawler_strategy: (Advanced) Provide a custom crawler strategy if needed. config: A BrowserConfig object specifying how the browser is set up. always_bypass_cache: (Deprecated) Use CrawlerRunConfig.cache_mode instead. base_directory: Folder for storing caches/logs (if relevant). thread_safe: If True, attempts some concurrency safeguards. Usually False. **kwargs: Additional legacy or debugging parameters. \"\"\" ) ### Typical Initialization ```python from crawl4ai import AsyncWebCrawler, BrowserConfig browser_cfg = BrowserConfig( browser_type=\"chromium\", headless=True, verbose=True ) crawler = AsyncWebCrawler(config=browser_cfg) `\n```\n\n**Notes** :\n\n * **Legacy** parameters like `always_bypass_cache` remain for backward compatibility, but prefer to set **caching** in `CrawlerRunConfig`.\n\n\n\n## 2. Lifecycle: Start/Close or Context Manager\n\n### 2.1 Context Manager (Recommended)\n\n```\n`async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun(\"https://example.com\") # The crawler automatically starts/closes resources `\n```\n\nWhen the `async with` block ends, the crawler cleans up (closes the browser, etc.).\n\n### 2.2 Manual Start & Close\n\n```\n`crawler = AsyncWebCrawler(config=browser_cfg) await crawler.start() result1 = await crawler.arun(\"https://example.com\") result2 = await crawler.arun(\"https://another.com\") await crawler.close() `\n```\n\nUse this style if you have a **long-running** application or need full control of the crawler’s lifecycle.\n\n## 3. Primary Method: `arun()`\n\n```\n`async def arun( self, url: str, config: Optional[CrawlerRunConfig] = None, # Legacy parameters for backward compatibility... ) -> CrawlResult: ... `\n```\n\n### 3.1 New Approach\n\nYou pass a `CrawlerRunConfig` object that sets up everything about a crawl—content filtering, caching, session reuse, JS code, screenshots, etc.\n\n```\n`import asyncio from crawl4ai import CrawlerRunConfig, CacheMode run_cfg = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, css_selector=\"main.article\", word_count_threshold=10, screenshot=True ) async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun(\"https://example.com/news\", config=run_cfg) print(\"Crawled HTML length:\", len(result.cleaned_html)) if result.screenshot: print(\"Screenshot base64 length:\", len(result.screenshot)) `\n```\n\n### 3.2 Legacy Parameters Still Accepted\n\nFor **backward** compatibility, `arun()` can still accept direct arguments like `css_selector=...`, `word_count_threshold=...`, etc., but we strongly advise migrating them into a **`CrawlerRunConfig`**.\n\n## 4. Batch Processing: `arun_many()`\n\n```\n`async def arun_many( self, urls: List[str], config: Optional[CrawlerRunConfig] = None, # Legacy parameters maintained for backwards compatibility... ) -> List[CrawlResult]: \"\"\" Process multiple URLs with intelligent rate limiting and resource monitoring. \"\"\" `\n```\n\n### 4.1 Resource-Aware Crawling\n\nThe `arun_many()` method now uses an intelligent dispatcher that:\n\n * Monitors system memory usage\n * Implements adaptive rate limiting\n * Provides detailed progress monitoring\n * Manages concurrent crawls efficiently\n\n\n\n### 4.2 Example Usage\n\nCheck page [Multi-url Crawling](../../advanced/multi-url-crawling/) for a detailed example of how to use `arun_many()`.\n\n```\n`### 4.3 Key Features 1. **Rate Limiting** - Automatic delay between requests - Exponential backoff on rate limit detection - Domain-specific rate limiting - Configurable retry strategy 2. **Resource Monitoring** - Memory usage tracking - Adaptive concurrency based on system load - Automatic pausing when resources are constrained 3. **Progress Monitoring** - Detailed or aggregated progress display - Real-time status updates - Memory usage statistics 4. **Error Handling** - Graceful handling of rate limits - Automatic retries with backoff - Detailed error reporting --- ## 5. `CrawlResult` Output Each `arun()` returns a **`CrawlResult`** containing: - `url`: Final URL (if redirected). - `html`: Original HTML. - `cleaned_html`: Sanitized HTML. - `markdown_v2`: Deprecated. Instead just use regular `markdown` - `extracted_content`: If an extraction strategy was used (JSON for CSS/LLM strategies). - `screenshot`, `pdf`: If screenshots/PDF requested. - `media`, `links`: Information about discovered images/links. - `success`, `error_message`: Status info. For details, see [CrawlResult doc](./crawl-result.md). --- ## 6. Quick Example Below is an example hooking it all together: ```python import asyncio from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode from crawl4ai.extraction_strategy import JsonCssExtractionStrategy import json async def main(): # 1. Browser config browser_cfg = BrowserConfig( browser_type=\"firefox\", headless=False, verbose=True ) # 2. Run config schema = { \"name\": \"Articles\", \"baseSelector\": \"article.post\", \"fields\": [ { \"name\": \"title\", \"selector\": \"h2\", \"type\": \"text\" }, { \"name\": \"url\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\" } ] } run_cfg = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, extraction_strategy=JsonCssExtractionStrategy(schema), word_count_threshold=15, remove_overlay_elements=True, wait_for=\"css:.post\" # Wait for posts to appear ) async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun( url=\"https://example.com/blog\", config=run_cfg ) if result.success: print(\"Cleaned HTML length:\", len(result.cleaned_html)) if result.extracted_content: articles = json.loads(result.extracted_content) print(\"Extracted articles:\", articles[:2]) else: print(\"Error:\", result.error_message) asyncio.run(main()) `\n```\n\n**Explanation** :\n\n * We define a **`BrowserConfig`** with Firefox, no headless, and `verbose=True`. \n * We define a **`CrawlerRunConfig`** that **bypasses cache** , uses a **CSS** extraction schema, has a `word_count_threshold=15`, etc. \n * We pass them to `AsyncWebCrawler(config=...)` and `arun(url=..., config=...)`.\n\n\n\n## 7. Best Practices & Migration Notes\n\n1. **Use** `BrowserConfig` for **global** settings about the browser’s environment. 2. **Use** `CrawlerRunConfig` for **per-crawl** logic (caching, content filtering, extraction strategies, wait conditions). 3. **Avoid** legacy parameters like `css_selector` or `word_count_threshold` directly in `arun()`. Instead:\n\n```\n`run_cfg = CrawlerRunConfig(css_selector=\".main-content\", word_count_threshold=20) result = await crawler.arun(url=\"...\", config=run_cfg) `\n```\n\n4. **Context Manager** usage is simplest unless you want a persistent crawler across many calls.\n\n## 8. Summary\n\n**AsyncWebCrawler** is your entry point to asynchronous crawling:\n\n * **Constructor** accepts **`BrowserConfig`** (or defaults). \n * **`arun(url, config=CrawlerRunConfig)`** is the main method for single-page crawls. \n * **`arun_many(urls, config=CrawlerRunConfig)`** handles concurrency across multiple URLs. \n * For advanced lifecycle control, use `start()` and `close()` explicitly. \n\n\n\n**Migration** : \n\n * If you used `AsyncWebCrawler(browser_type=\"chromium\", css_selector=\"...\")`, move browser settings to `BrowserConfig(...)` and content/crawl logic to `CrawlerRunConfig(...)`.\n\n\n\nThis modular approach ensures your code is **clean** , **scalable** , and **easy to maintain**. For any advanced or rarely used parameters, see the [BrowserConfig docs](../parameters/).\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../async-webcrawler/)\n * [arun()](../arun/)\n * [arun_many()](../arun_many/)\n * [Browser, Crawler & LLM Config](../parameters/)\n * CrawlResult\n * [Strategies](../strategies/)\n\n\n\n * [CrawlResult Reference](#crawlresult-reference)\n * [1. Basic Crawl Info](#1-basic-crawl-info)\n * [2. Raw / Cleaned Content](#2-raw-cleaned-content)\n * [3. Markdown Fields](#3-markdown-fields)\n * [4. Media & Links](#4-media-links)\n * [5. Additional Fields](#5-additional-fields)\n * [6. dispatch_result (optional)](#6-dispatch_result-optional)\n * [7. Example: Accessing Everything](#7-example-accessing-everything)\n * [8. Key Points & Future](#8-key-points-future)\n\n\n\n# `CrawlResult` Reference\n\nThe **`CrawlResult`** class encapsulates everything returned after a single crawl operation. It provides the **raw or processed content** , details on links and media, plus optional metadata (like screenshots, PDFs, or extracted JSON).\n\n**Location** : `crawl4ai/crawler/models.py` (for reference)\n\n```\n`class CrawlResult(BaseModel): url: str html: str success: bool cleaned_html: Optional[str] = None media: Dict[str, List[Dict]] = {} links: Dict[str, List[Dict]] = {} downloaded_files: Optional[List[str]] = None screenshot: Optional[str] = None pdf : Optional[bytes] = None markdown: Optional[Union[str, MarkdownGenerationResult]] = None extracted_content: Optional[str] = None metadata: Optional[dict] = None error_message: Optional[str] = None session_id: Optional[str] = None response_headers: Optional[dict] = None status_code: Optional[int] = None ssl_certificate: Optional[SSLCertificate] = None dispatch_result: Optional[DispatchResult] = None ... `\n```\n\nBelow is a **field-by-field** explanation and possible usage patterns.\n\n## 1. Basic Crawl Info\n\n### 1.1 **`url`** _(str)_\n\n**What** : The final crawled URL (after any redirects). **Usage** : \n\n```\n`print(result.url) # e.g., \"https://example.com/\" `\n```\n\n### 1.2 **`success`** _(bool)_\n\n**What** : `True` if the crawl pipeline ended without major errors; `False` otherwise. **Usage** : \n\n```\n`if not result.success: print(f\"Crawl failed: {result.error_message}\") `\n```\n\n### 1.3 **`status_code`** _(Optional[int])_\n\n**What** : The page’s HTTP status code (e.g., 200, 404). **Usage** : \n\n```\n`if result.status_code == 404: print(\"Page not found!\") `\n```\n\n### 1.4 **`error_message`** _(Optional[str])_\n\n**What** : If `success=False`, a textual description of the failure. **Usage** : \n\n```\n`if not result.success: print(\"Error:\", result.error_message) `\n```\n\n### 1.5 **`session_id`** _(Optional[str])_\n\n**What** : The ID used for reusing a browser context across multiple calls. **Usage** : \n\n```\n`# If you used session_id=\"login_session\" in CrawlerRunConfig, see it here: print(\"Session:\", result.session_id) `\n```\n\n### 1.6 **`response_headers`** _(Optional[dict])_\n\n**What** : Final HTTP response headers. **Usage** : \n\n```\n`if result.response_headers: print(\"Server:\", result.response_headers.get(\"Server\", \"Unknown\")) `\n```\n\n### 1.7 **`ssl_certificate`** _(Optional[SSLCertificate])_\n\n**What** : If `fetch_ssl_certificate=True` in your CrawlerRunConfig, **`result.ssl_certificate`** contains a [**`SSLCertificate`**](../../advanced/ssl-certificate/) object describing the site’s certificate. You can export the cert in multiple formats (PEM/DER/JSON) or access its properties like `issuer`, `subject`, `valid_from`, `valid_until`, etc. **Usage** : \n\n```\n`if result.ssl_certificate: print(\"Issuer:\", result.ssl_certificate.issuer) `\n```\n\n## 2. Raw / Cleaned Content\n\n### 2.1 **`html`** _(str)_\n\n**What** : The **original** unmodified HTML from the final page load. **Usage** : \n\n```\n`# Possibly large print(len(result.html)) `\n```\n\n### 2.2 **`cleaned_html`** _(Optional[str])_\n\n**What** : A sanitized HTML version—scripts, styles, or excluded tags are removed based on your `CrawlerRunConfig`. **Usage** : \n\n```\n`print(result.cleaned_html[:500]) # Show a snippet `\n```\n\n### 2.3 **`fit_html`** _(Optional[str])_\n\n**What** : If a **content filter** or heuristic (e.g., Pruning/BM25) modifies the HTML, the “fit” or post-filter version. **When** : This is **only** present if your `markdown_generator` or `content_filter` produces it. **Usage** : \n\n```\n`if result.markdown.fit_html: print(\"High-value HTML content:\", result.markdown.fit_html[:300]) `\n```\n\n## 3. Markdown Fields\n\n### 3.1 The Markdown Generation Approach\n\nCrawl4AI can convert HTML→Markdown, optionally including:\n\n * **Raw** markdown \n * **Links as citations** (with a references section) \n * **Fit** markdown if a **content filter** is used (like Pruning or BM25)\n\n\n\n**`MarkdownGenerationResult`** includes: - **`raw_markdown`** _(str)_ : The full HTML→Markdown conversion. - **`markdown_with_citations`** _(str)_ : Same markdown, but with link references as academic-style citations. - **`references_markdown`** _(str)_ : The reference list or footnotes at the end. - **`fit_markdown`** _(Optional[str])_ : If content filtering (Pruning/BM25) was applied, the filtered “fit” text. - **`fit_html`** _(Optional[str])_ : The HTML that led to `fit_markdown`.\n\n**Usage** : \n\n```\n`if result.markdown: md_res = result.markdown print(\"Raw MD:\", md_res.raw_markdown[:300]) print(\"Citations MD:\", md_res.markdown_with_citations[:300]) print(\"References:\", md_res.references_markdown) if md_res.fit_markdown: print(\"Pruned text:\", md_res.fit_markdown[:300]) `\n```\n\n### 3.2 **`markdown`** _(Optional[Union[str, MarkdownGenerationResult]])_\n\n**What** : Holds the `MarkdownGenerationResult`. **Usage** : \n\n```\n`print(result.markdown.raw_markdown[:200]) print(result.markdown.fit_markdown) print(result.markdown.fit_html) `\n```\n\n**Important** : “Fit” content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`. \n\n## 4. Media & Links\n\n### 4.1 **`media`** _(Dict[str, List[Dict]])_\n\n**What** : Contains info about discovered images, videos, or audio. Typically keys: `\"images\"`, `\"videos\"`, `\"audios\"`. **Common Fields** in each item:\n\n * `src` _(str)_ : Media URL \n * `alt` or `title` _(str)_ : Descriptive text \n * `score` _(float)_ : Relevance score if the crawler’s heuristic found it “important” \n * `desc` or `description` _(Optional[str])_ : Additional context extracted from surrounding text \n\n\n\n**Usage** : \n\n```\n`images = result.media.get(\"images\", []) for img in images: if img.get(\"score\", 0) > 5: print(\"High-value image:\", img[\"src\"]) `\n```\n\n### 4.2 **`links`** _(Dict[str, List[Dict]])_\n\n**What** : Holds internal and external link data. Usually two keys: `\"internal\"` and `\"external\"`. **Common Fields** :\n\n * `href` _(str)_ : The link target \n * `text` _(str)_ : Link text \n * `title` _(str)_ : Title attribute \n * `context` _(str)_ : Surrounding text snippet \n * `domain` _(str)_ : If external, the domain\n\n\n\n**Usage** : \n\n```\n`for link in result.links[\"internal\"]: print(f\"Internal link to {link['href']} with text {link['text']}\") `\n```\n\n## 5. Additional Fields\n\n### 5.1 **`extracted_content`** _(Optional[str])_\n\n**What** : If you used **`extraction_strategy`** (CSS, LLM, etc.), the structured output (JSON). **Usage** : \n\n```\n`if result.extracted_content: data = json.loads(result.extracted_content) print(data) `\n```\n\n### 5.2 **`downloaded_files`** _(Optional[List[str]])_\n\n**What** : If `accept_downloads=True` in your `BrowserConfig` + `downloads_path`, lists local file paths for downloaded items. **Usage** : \n\n```\n`if result.downloaded_files: for file_path in result.downloaded_files: print(\"Downloaded:\", file_path) `\n```\n\n### 5.3 **`screenshot`** _(Optional[str])_\n\n**What** : Base64-encoded screenshot if `screenshot=True` in `CrawlerRunConfig`. **Usage** : \n\n```\n`import base64 if result.screenshot: with open(\"page.png\", \"wb\") as f: f.write(base64.b64decode(result.screenshot)) `\n```\n\n### 5.4 **`pdf`** _(Optional[bytes])_\n\n**What** : Raw PDF bytes if `pdf=True` in `CrawlerRunConfig`. **Usage** : \n\n```\n`if result.pdf: with open(\"page.pdf\", \"wb\") as f: f.write(result.pdf) `\n```\n\n### 5.5 **`metadata`** _(Optional[dict])_\n\n**What** : Page-level metadata if discovered (title, description, OG data, etc.). **Usage** : \n\n```\n`if result.metadata: print(\"Title:\", result.metadata.get(\"title\")) print(\"Author:\", result.metadata.get(\"author\")) `\n```\n\n## 6. `dispatch_result` (optional)\n\nA `DispatchResult` object providing additional concurrency and resource usage information when crawling URLs in parallel (e.g., via `arun_many()` with custom dispatchers). It contains:\n\n * **`task_id`** : A unique identifier for the parallel task.\n * **`memory_usage`** (float): The memory (in MB) used at the time of completion.\n * **`peak_memory`** (float): The peak memory usage (in MB) recorded during the task’s execution.\n * **`start_time`** / **`end_time`** (datetime): Time range for this crawling task.\n * **`error_message`** (str): Any dispatcher- or concurrency-related error encountered.\n\n\n\n```\n`# Example usage: for result in results: if result.success and result.dispatch_result: dr = result.dispatch_result print(f\"URL: {result.url}, Task ID: {dr.task_id}\") print(f\"Memory: {dr.memory_usage:.1f} MB (Peak: {dr.peak_memory:.1f} MB)\") print(f\"Duration: {dr.end_time - dr.start_time}\") `\n```\n\n> **Note** : This field is typically populated when using `arun_many(...)` alongside a **dispatcher** (e.g., `MemoryAdaptiveDispatcher` or `SemaphoreDispatcher`). If no concurrency or dispatcher is used, `dispatch_result` may remain `None`. \n\n## 7. Example: Accessing Everything\n\n```\n`async def handle_result(result: CrawlResult): if not result.success: print(\"Crawl error:\", result.error_message) return # Basic info print(\"Crawled URL:\", result.url) print(\"Status code:\", result.status_code) # HTML print(\"Original HTML size:\", len(result.html)) print(\"Cleaned HTML size:\", len(result.cleaned_html or \"\")) # Markdown output if result.markdown: print(\"Raw Markdown:\", result.markdown.raw_markdown[:300]) print(\"Citations Markdown:\", result.markdown.markdown_with_citations[:300]) if result.markdown.fit_markdown: print(\"Fit Markdown:\", result.markdown.fit_markdown[:200]) # Media & Links if \"images\" in result.media: print(\"Image count:\", len(result.media[\"images\"])) if \"internal\" in result.links: print(\"Internal link count:\", len(result.links[\"internal\"])) # Extraction strategy result if result.extracted_content: print(\"Structured data:\", result.extracted_content) # Screenshot/PDF if result.screenshot: print(\"Screenshot length:\", len(result.screenshot)) if result.pdf: print(\"PDF bytes length:\", len(result.pdf)) `\n```\n\n## 8. Key Points & Future\n\n1. **Deprecated legacy properties of CrawlResult** - `markdown_v2` - Deprecated in v0.5. Just use `markdown`. It holds the `MarkdownGenerationResult` now! - `fit_markdown` and `fit_html` - Deprecated in v0.5. They can now be accessed via `MarkdownGenerationResult` in `result.markdown`. eg: `result.markdown.fit_markdown` and `result.markdown.fit_html`\n\n2. **Fit Content** - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly. - If no filter is used, they remain `None`.\n\n3. **References & Citations** - If you enable link citations in your `DefaultMarkdownGenerator` (`options={\"citations\": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing.\n\n4. **Links & Media** - `links[\"internal\"]` and `links[\"external\"]` group discovered anchors by domain. - `media[\"images\"]` / `[\"videos\"]` / `[\"audios\"]` store extracted media elements with optional scoring or context.\n\n5. **Error Cases** - If `success=False`, check `error_message` (e.g., timeouts, invalid URLs). - `status_code` might be `None` if we failed before an HTTP response.\n\nUse **`CrawlResult`** to glean all final outputs and feed them into your data pipelines, AI models, or archives. With the synergy of a properly configured **BrowserConfig** and **CrawlerRunConfig** , the crawler can produce robust, structured results here in **`CrawlResult`**.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../async-webcrawler/)\n * [arun()](../arun/)\n * [arun_many()](../arun_many/)\n * Browser, Crawler & LLM Config\n * [CrawlResult](../crawl-result/)\n * [Strategies](../strategies/)\n\n\n\n * [1. BrowserConfig – Controlling the Browser](#1-browserconfig-controlling-the-browser)\n * [1.1 Parameter Highlights](#11-parameter-highlights)\n * [2. CrawlerRunConfig – Controlling Each Crawl](#2-crawlerrunconfig-controlling-each-crawl)\n * [2.1 Parameter Highlights](#21-parameter-highlights)\n * [2.2 Helper Methods](#22-helper-methods)\n * [2.3 Example Usage](#23-example-usage)\n * [3. LLMConfig - Setting up LLM providers](#3-llmconfig-setting-up-llm-providers)\n * [3.1 Parameters](#31-parameters)\n * [3.2 Example Usage](#32-example-usage)\n * [4. Putting It All Together](#4-putting-it-all-together)\n\n\n\n# 1. **BrowserConfig** – Controlling the Browser\n\n`BrowserConfig` focuses on **how** the browser is launched and behaves. This includes headless mode, proxies, user agents, and other environment tweaks.\n\n```\n`from crawl4ai import AsyncWebCrawler, BrowserConfig browser_cfg = BrowserConfig( browser_type=\"chromium\", headless=True, viewport_width=1280, viewport_height=720, proxy=\"http://user:pass@proxy:8080\", user_agent=\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36\", ) `\n```\n\n## 1.1 Parameter Highlights\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`browser_type`** | `\"chromium\"`, `\"firefox\"`, `\"webkit\"`_(default:`\"chromium\"`)_ | Which browser engine to use. `\"chromium\"` is typical for many sites, `\"firefox\"` or `\"webkit\"` for specialized tests. \n**`headless`** | `bool` (default: `True`) | Headless means no visible UI. `False` is handy for debugging. \n**`viewport_width`** | `int` (default: `1080`) | Initial page width (in px). Useful for testing responsive layouts. \n**`viewport_height`** | `int` (default: `600`) | Initial page height (in px). \n**`proxy`** | `str` (default: `None`) | Single-proxy URL if you want all traffic to go through it, e.g. `\"http://user:pass@proxy:8080\"`. \n**`proxy_config`** | `dict` (default: `None`) | For advanced or multi-proxy needs, specify details like `{\"server\": \"...\", \"username\": \"...\", ...}`. \n**`use_persistent_context`** | `bool` (default: `False`) | If `True`, uses a **persistent** browser context (keep cookies, sessions across runs). Also sets `use_managed_browser=True`. \n**`user_data_dir`** | `str or None` (default: `None`) | Directory to store user data (profiles, cookies). Must be set if you want permanent sessions. \n**`ignore_https_errors`** | `bool` (default: `True`) | If `True`, continues despite invalid certificates (common in dev/staging). \n**`java_script_enabled`** | `bool` (default: `True`) | Disable if you want no JS overhead, or if only static content is needed. \n**`cookies`** | `list` (default: `[]`) | Pre-set cookies, each a dict like `{\"name\": \"session\", \"value\": \"...\", \"url\": \"...\"}`. \n**`headers`** | `dict` (default: `{}`) | Extra HTTP headers for every request, e.g. `{\"Accept-Language\": \"en-US\"}`. \n**`user_agent`** | `str` (default: Chrome-based UA) | Your custom or random user agent. `user_agent_mode=\"random\"` can shuffle it. \n**`light_mode`** | `bool` (default: `False`) | Disables some background features for performance gains. \n**`text_mode`** | `bool` (default: `False`) | If `True`, tries to disable images/other heavy content for speed. \n**`use_managed_browser`** | `bool` (default: `False`) | For advanced “managed” interactions (debugging, CDP usage). Typically set automatically if persistent context is on. \n**`extra_args`** | `list` (default: `[]`) | Additional flags for the underlying browser process, e.g. `[\"--disable-extensions\"]`. \n \n**Tips** : - Set `headless=False` to visually **debug** how pages load or how interactions proceed. - If you need **authentication** storage or repeated sessions, consider `use_persistent_context=True` and specify `user_data_dir`. - For large pages, you might need a bigger `viewport_width` and `viewport_height` to handle dynamic content.\n\n# 2. **CrawlerRunConfig** – Controlling Each Crawl\n\nWhile `BrowserConfig` sets up the **environment** , `CrawlerRunConfig` details **how** each **crawl operation** should behave: caching, content filtering, link or domain blocking, timeouts, JavaScript code, etc.\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig run_cfg = CrawlerRunConfig( wait_for=\"css:.main-content\", word_count_threshold=15, excluded_tags=[\"nav\", \"footer\"], exclude_external_links=True, stream=True, # Enable streaming for arun_many() ) `\n```\n\n## 2.1 Parameter Highlights\n\nWe group them by category. \n\n### A) **Content Processing**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`word_count_threshold`** | `int` (default: ~200) | Skips text blocks below X words. Helps ignore trivial sections. \n**`extraction_strategy`** | `ExtractionStrategy` (default: None) | If set, extracts structured data (CSS-based, LLM-based, etc.). \n**`markdown_generator`** | `MarkdownGenerationStrategy` (None) | If you want specialized markdown output (citations, filtering, chunking, etc.). \n**`css_selector`** | `str` (None) | Retains only the part of the page matching this selector. Affects the entire extraction process. \n**`target_elements`** | `List[str]` (None) | List of CSS selectors for elements to focus on for markdown generation and data extraction, while still processing the entire page for links, media, etc. Provides more flexibility than `css_selector`. \n**`excluded_tags`** | `list` (None) | Removes entire tags (e.g. `[\"script\", \"style\"]`). \n**`excluded_selector`** | `str` (None) | Like `css_selector` but to exclude. E.g. `\"#ads, .tracker\"`. \n**`only_text`** | `bool` (False) | If `True`, tries to extract text-only content. \n**`prettiify`** | `bool` (False) | If `True`, beautifies final HTML (slower, purely cosmetic). \n**`keep_data_attributes`** | `bool` (False) | If `True`, preserve `data-*` attributes in cleaned HTML. \n**`remove_forms`** | `bool` (False) | If `True`, remove all `` elements. \n \n### B) **Caching & Session**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`cache_mode`** | `CacheMode or None` | Controls how caching is handled (`ENABLED`, `BYPASS`, `DISABLED`, etc.). If `None`, typically defaults to `ENABLED`. \n**`session_id`** | `str or None` | Assign a unique ID to reuse a single browser session across multiple `arun()` calls. \n**`bypass_cache`** | `bool` (False) | If `True`, acts like `CacheMode.BYPASS`. \n**`disable_cache`** | `bool` (False) | If `True`, acts like `CacheMode.DISABLED`. \n**`no_cache_read`** | `bool` (False) | If `True`, acts like `CacheMode.WRITE_ONLY` (writes cache but never reads). \n**`no_cache_write`** | `bool` (False) | If `True`, acts like `CacheMode.READ_ONLY` (reads cache but never writes). \n \nUse these for controlling whether you read or write from a local content cache. Handy for large batch crawls or repeated site visits.\n\n### C) **Page Navigation & Timing**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`wait_until`** | `str` (domcontentloaded) | Condition for navigation to “complete”. Often `\"networkidle\"` or `\"domcontentloaded\"`. \n**`page_timeout`** | `int` (60000 ms) | Timeout for page navigation or JS steps. Increase for slow sites. \n**`wait_for`** | `str or None` | Wait for a CSS (`\"css:selector\"`) or JS (`\"js:() => bool\"`) condition before content extraction. \n**`wait_for_images`** | `bool` (False) | Wait for images to load before finishing. Slows down if you only want text. \n**`delay_before_return_html`** | `float` (0.1) | Additional pause (seconds) before final HTML is captured. Good for last-second updates. \n**`check_robots_txt`** | `bool` (False) | Whether to check and respect robots.txt rules before crawling. If True, caches robots.txt for efficiency. \n**`mean_delay`** and **`max_range`** | `float` (0.1, 0.3) | If you call `arun_many()`, these define random delay intervals between crawls, helping avoid detection or rate limits. \n**`semaphore_count`** | `int` (5) | Max concurrency for `arun_many()`. Increase if you have resources for parallel crawls. \n \n### D) **Page Interaction**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`js_code`** | `str or list[str]` (None) | JavaScript to run after load. E.g. `\"document.querySelector('button')?.click();\"`. \n**`js_only`** | `bool` (False) | If `True`, indicates we’re reusing an existing session and only applying JS. No full reload. \n**`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. \n**`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). \n**`scroll_delay`** | `float` (0.2) | Delay between scroll steps if `scan_full_page=True`. \n**`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. \n**`remove_overlay_elements`** | `bool` (False) | Removes potential modals/popups blocking the main content. \n**`simulate_user`** | `bool` (False) | Simulate user interactions (mouse movements) to avoid bot detection. \n**`override_navigator`** | `bool` (False) | Override `navigator` properties in JS for stealth. \n**`magic`** | `bool` (False) | Automatic handling of popups/consent banners. Experimental. \n**`adjust_viewport_to_content`** | `bool` (False) | Resizes viewport to match page content height. \n \nIf your page is a single-page app with repeated JS updates, set `js_only=True` in subsequent calls, plus a `session_id` for reusing the same tab.\n\n### E) **Media Handling**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`screenshot`** | `bool` (False) | Capture a screenshot (base64) in `result.screenshot`. \n**`screenshot_wait_for`** | `float or None` | Extra wait time before the screenshot. \n**`screenshot_height_threshold`** | `int` (~20000) | If the page is taller than this, alternate screenshot strategies are used. \n**`pdf`** | `bool` (False) | If `True`, returns a PDF in `result.pdf`. \n**`image_description_min_word_threshold`** | `int` (~50) | Minimum words for an image’s alt text or description to be considered valid. \n**`image_score_threshold`** | `int` (~3) | Filter out low-scoring images. The crawler scores images by relevance (size, context, etc.). \n**`exclude_external_images`** | `bool` (False) | Exclude images from other domains. \n \n### F) **Link/Domain Handling**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`exclude_social_media_domains`** | `list` (e.g. Facebook/Twitter) | A default list can be extended. Any link to these domains is removed from final output. \n**`exclude_external_links`** | `bool` (False) | Removes all links pointing outside the current domain. \n**`exclude_social_media_links`** | `bool` (False) | Strips links specifically to social sites (like Facebook or Twitter). \n**`exclude_domains`** | `list` ([]) | Provide a custom list of domains to exclude (like `[\"ads.com\", \"trackers.io\"]`). \n \nUse these for link-level content filtering (often to keep crawls “internal” or to remove spammy domains).\n\n### G) **Debug & Logging**\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`verbose`** | `bool` (True) | Prints logs detailing each step of crawling, interactions, or errors. \n**`log_console`** | `bool` (False) | Logs the page’s JavaScript console output if you want deeper JS debugging. \n \n## 2.2 Helper Methods\n\nBoth `BrowserConfig` and `CrawlerRunConfig` provide a `clone()` method to create modified copies:\n\n```\n`# Create a base configuration base_config = CrawlerRunConfig( cache_mode=CacheMode.ENABLED, word_count_threshold=200 ) # Create variations using clone() stream_config = base_config.clone(stream=True) no_cache_config = base_config.clone( cache_mode=CacheMode.BYPASS, stream=True ) `\n```\n\nThe `clone()` method is particularly useful when you need slightly different configurations for different use cases, without modifying the original config.\n\n## 2.3 Example Usage\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode async def main(): # Configure the browser browser_cfg = BrowserConfig( headless=False, viewport_width=1280, viewport_height=720, proxy=\"http://user:pass@myproxy:8080\", text_mode=True ) # Configure the run run_cfg = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, session_id=\"my_session\", css_selector=\"main.article\", excluded_tags=[\"script\", \"style\"], exclude_external_links=True, wait_for=\"css:.article-loaded\", screenshot=True, stream=True ) async with AsyncWebCrawler(config=browser_cfg) as crawler: result = await crawler.arun( url=\"https://example.com/news\", config=run_cfg ) if result.success: print(\"Final cleaned_html length:\", len(result.cleaned_html)) if result.screenshot: print(\"Screenshot captured (base64, length):\", len(result.screenshot)) else: print(\"Crawl failed:\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) ## 2.4 Compliance & Ethics | **Parameter** | **Type / Default** | **What It Does** | |-----------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------| | **`check_robots_txt`**| `bool` (False) | When True, checks and respects robots.txt rules before crawling. Uses efficient caching with SQLite backend. | | **`user_agent`** | `str` (None) | User agent string to identify your crawler. Used for robots.txt checking when enabled. | ```python run_config = CrawlerRunConfig( check_robots_txt=True, # Enable robots.txt compliance user_agent=\"MyBot/1.0\" # Identify your crawler ) `\n```\n\n# 3. **LLMConfig** - Setting up LLM providers\n\nLLMConfig is useful to pass LLM provider config to strategies and functions that rely on LLMs to do extraction, filtering, schema generation etc. Currently it can be used in the following -\n\n 1. LLMExtractionStrategy\n 2. LLMContentFilter\n 3. JsonCssExtractionStrategy.generate_schema\n 4. JsonXPathExtractionStrategy.generate_schema\n\n\n\n## 3.1 Parameters\n\n**Parameter** | **Type / Default** | **What It Does** \n---|---|--- \n**`provider`** | `\"ollama/llama3\",\"groq/llama3-70b-8192\",\"groq/llama3-8b-8192\", \"openai/gpt-4o-mini\" ,\"openai/gpt-4o\",\"openai/o1-mini\",\"openai/o1-preview\",\"openai/o3-mini\",\"openai/o3-mini-high\",\"anthropic/claude-3-haiku-20240307\",\"anthropic/claude-3-opus-20240229\",\"anthropic/claude-3-sonnet-20240229\",\"anthropic/claude-3-5-sonnet-20240620\",\"gemini/gemini-pro\",\"gemini/gemini-1.5-pro\",\"gemini/gemini-2.0-flash\",\"gemini/gemini-2.0-flash-exp\",\"gemini/gemini-2.0-flash-lite-preview-02-05\",\"deepseek/deepseek-chat\"`_(default:`\"openai/gpt-4o-mini\"`)_ | Which LLM provoder to use. \n**`api_token`** | 1.Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`\"GEMINI_API_KEY\"` will be read from environment variables 2. API token of LLM provider eg: `api_token = \"gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv\"` 3. Environment variable - use with prefix \"env:\" eg:`api_token = \"env: GROQ_API_KEY\"` | API token to use for the given provider \n**`base_url`** | Optional. Custom API endpoint | If your provider has a custom endpoint \n \n## 3.2 Example Usage\n\n```\n`llm_config = LLMConfig(provider=\"openai/gpt-4o-mini\", api_token=os.getenv(\"OPENAI_API_KEY\")) `\n```\n\n## 4. Putting It All Together\n\n * **Use** `BrowserConfig` for **global** browser settings: engine, headless, proxy, user agent. \n * **Use** `CrawlerRunConfig` for each crawl’s **context** : how to filter content, handle caching, wait for dynamic elements, or run JS. \n * **Pass** both configs to `AsyncWebCrawler` (the `BrowserConfig`) and then to `arun()` (the `CrawlerRunConfig`). \n * **Use** `LLMConfig` for LLM provider configurations that can be used across all extraction, filtering, schema generation tasks. Can be used in - `LLMExtractionStrategy`, `LLMContentFilter`, `JsonCssExtractionStrategy.generate_schema` & `JsonXPathExtractionStrategy.generate_schema`\n\n\n\n```\n`# Create a modified copy with the clone() method stream_cfg = run_cfg.clone( stream=True, cache_mode=CacheMode.BYPASS ) `\n```\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../async-webcrawler/)\n * [arun()](../arun/)\n * [arun_many()](../arun_many/)\n * [Browser, Crawler & LLM Config](../parameters/)\n * [CrawlResult](../crawl-result/)\n * Strategies\n\n\n\n * [Extraction & Chunking Strategies API](#extraction-chunking-strategies-api)\n * [Extraction Strategies](#extraction-strategies)\n * [Chunking Strategies](#chunking-strategies)\n * [Usage Examples](#usage-examples)\n * [Best Practices](#best-practices)\n\n\n\n# Extraction & Chunking Strategies API\n\nThis documentation covers the API reference for extraction and chunking strategies in Crawl4AI.\n\n## Extraction Strategies\n\nAll extraction strategies inherit from the base `ExtractionStrategy` class and implement two key methods: - `extract(url: str, html: str) -> List[Dict[str, Any]]` - `run(url: str, sections: List[str]) -> List[Dict[str, Any]]`\n\n### LLMExtractionStrategy\n\nUsed for extracting structured data using Language Models.\n\n```\n`LLMExtractionStrategy( # Required Parameters provider: str = DEFAULT_PROVIDER, # LLM provider (e.g., \"ollama/llama2\") api_token: Optional[str] = None, # API token # Extraction Configuration instruction: str = None, # Custom extraction instruction schema: Dict = None, # Pydantic model schema for structured data extraction_type: str = \"block\", # \"block\" or \"schema\" # Chunking Parameters chunk_token_threshold: int = 4000, # Maximum tokens per chunk overlap_rate: float = 0.1, # Overlap between chunks word_token_rate: float = 0.75, # Word to token conversion rate apply_chunking: bool = True, # Enable/disable chunking # API Configuration base_url: str = None, # Base URL for API extra_args: Dict = {}, # Additional provider arguments verbose: bool = False # Enable verbose logging ) `\n```\n\n### CosineStrategy\n\nUsed for content similarity-based extraction and clustering.\n\n```\n`CosineStrategy( # Content Filtering semantic_filter: str = None, # Topic/keyword filter word_count_threshold: int = 10, # Minimum words per cluster sim_threshold: float = 0.3, # Similarity threshold # Clustering Parameters max_dist: float = 0.2, # Maximum cluster distance linkage_method: str = 'ward', # Clustering method top_k: int = 3, # Top clusters to return # Model Configuration model_name: str = 'sentence-transformers/all-MiniLM-L6-v2', # Embedding model verbose: bool = False # Enable verbose logging ) `\n```\n\n### JsonCssExtractionStrategy\n\nUsed for CSS selector-based structured data extraction.\n\n```\n`JsonCssExtractionStrategy( schema: Dict[str, Any], # Extraction schema verbose: bool = False # Enable verbose logging ) # Schema Structure schema = { \"name\": str, # Schema name \"baseSelector\": str, # Base CSS selector \"fields\": [ # List of fields to extract { \"name\": str, # Field name \"selector\": str, # CSS selector \"type\": str, # Field type: \"text\", \"attribute\", \"html\", \"regex\" \"attribute\": str, # For type=\"attribute\" \"pattern\": str, # For type=\"regex\" \"transform\": str, # Optional: \"lowercase\", \"uppercase\", \"strip\" \"default\": Any # Default value if extraction fails } ] } `\n```\n\n## Chunking Strategies\n\nAll chunking strategies inherit from `ChunkingStrategy` and implement the `chunk(text: str) -> list` method.\n\n### RegexChunking\n\nSplits text based on regex patterns.\n\n```\n`RegexChunking( patterns: List[str] = None # Regex patterns for splitting # Default: [r'\\n\\n'] ) `\n```\n\n### SlidingWindowChunking\n\nCreates overlapping chunks with a sliding window approach.\n\n```\n`SlidingWindowChunking( window_size: int = 100, # Window size in words step: int = 50 # Step size between windows ) `\n```\n\n### OverlappingWindowChunking\n\nCreates chunks with specified overlap.\n\n```\n`OverlappingWindowChunking( window_size: int = 1000, # Chunk size in words overlap: int = 100 # Overlap size in words ) `\n```\n\n## Usage Examples\n\n### LLM Extraction\n\n```\n`from pydantic import BaseModel from crawl4ai.extraction_strategy import LLMExtractionStrategy from crawl4ai import LLMConfig # Define schema class Article(BaseModel): title: str content: str author: str # Create strategy strategy = LLMExtractionStrategy( llm_config = LLMConfig(provider=\"ollama/llama2\"), schema=Article.schema(), instruction=\"Extract article details\" ) # Use with crawler result = await crawler.arun( url=\"https://example.com/article\", extraction_strategy=strategy ) # Access extracted data data = json.loads(result.extracted_content) `\n```\n\n### CSS Extraction\n\n```\n`from crawl4ai.extraction_strategy import JsonCssExtractionStrategy # Define schema schema = { \"name\": \"Product List\", \"baseSelector\": \".product-card\", \"fields\": [ { \"name\": \"title\", \"selector\": \"h2.title\", \"type\": \"text\" }, { \"name\": \"price\", \"selector\": \".price\", \"type\": \"text\", \"transform\": \"strip\" }, { \"name\": \"image\", \"selector\": \"img\", \"type\": \"attribute\", \"attribute\": \"src\" } ] } # Create and use strategy strategy = JsonCssExtractionStrategy(schema) result = await crawler.arun( url=\"https://example.com/products\", extraction_strategy=strategy ) `\n```\n\n### Content Chunking\n\n```\n`from crawl4ai.chunking_strategy import OverlappingWindowChunking from crawl4ai import LLMConfig # Create chunking strategy chunker = OverlappingWindowChunking( window_size=500, # 500 words per chunk overlap=50 # 50 words overlap ) # Use with extraction strategy strategy = LLMExtractionStrategy( llm_config = LLMConfig(provider=\"ollama/llama2\"), chunking_strategy=chunker ) result = await crawler.arun( url=\"https://example.com/long-article\", extraction_strategy=strategy ) `\n```\n\n## Best Practices\n\n1. **Choose the Right Strategy** - Use `LLMExtractionStrategy` for complex, unstructured content - Use `JsonCssExtractionStrategy` for well-structured HTML - Use `CosineStrategy` for content similarity and clustering\n\n2. **Optimize Chunking**\n\n```\n`# For long documents strategy = LLMExtractionStrategy( chunk_token_threshold=2000, # Smaller chunks overlap_rate=0.1 # 10% overlap ) `\n```\n\n3. **Handle Errors**\n\n```\n`try: result = await crawler.arun( url=\"https://example.com\", extraction_strategy=strategy ) if result.success: content = json.loads(result.extracted_content) except Exception as e: print(f\"Extraction failed: {e}\") `\n```\n\n4. **Monitor Performance**\n\n```\n`strategy = CosineStrategy( verbose=True, # Enable logging word_count_threshold=20, # Filter short content top_k=5 # Limit results ) `\n```\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../../core/installation/)\n * [Docker Deployment](../../core/docker-deployment/)\n * [Quick Start](../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../core/cli/)\n * [Simple Crawling](../../core/simple-crawling/)\n * [Deep Crawling](../../core/deep-crawling/)\n * [Crawler Result](../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../core/browser-crawler-config/)\n * [Markdown Generation](../../core/markdown-generation/)\n * [Fit Markdown](../../core/fit-markdown/)\n * [Page Interaction](../../core/page-interaction/)\n * [Content Selection](../../core/content-selection/)\n * [Cache Modes](../../core/cache-modes/)\n * [Local Files & Raw HTML](../../core/local-files/)\n * [Link & Media](../../core/link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Installation 💻](#installation)\n * [Option 1: Python Package Installation (Recommended)](#option-1-python-package-installation-recommended)\n * [Playwright Installation Note for Ubuntu](#playwright-installation-note-for-ubuntu)\n * [Option 2: Using Docker (Coming Soon)](#option-2-using-docker-coming-soon)\n * [Option 3: Local Server Installation](#option-3-local-server-installation)\n * [Verifying Your Installation](#verifying-your-installation)\n * [Getting Help](#getting-help)\n\n\n\n# Installation 💻\n\nCrawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package, use it with Docker, or run it as a local server.\n\n## Option 1: Python Package Installation (Recommended)\n\nCrawl4AI is now available on PyPI, making installation easier than ever. Choose the option that best fits your needs:\n\n### Basic Installation\n\nFor basic web crawling and scraping tasks:\n\n```\n`pip install crawl4ai playwright install # Install Playwright dependencies `\n```\n\n### Installation with PyTorch\n\nFor advanced text clustering (includes CosineSimilarity cluster strategy):\n\n```\n`pip install crawl4ai[torch] `\n```\n\n### Installation with Transformers\n\nFor text summarization and Hugging Face models:\n\n```\n`pip install crawl4ai[transformer] `\n```\n\n### Full Installation\n\nFor all features:\n\n```\n`pip install crawl4ai[all] `\n```\n\n### Development Installation\n\nFor contributors who plan to modify the source code:\n\n```\n`git clone https://github.com/unclecode/crawl4ai.git cd crawl4ai pip install -e \".[all]\" playwright install # Install Playwright dependencies `\n```\n\n💡 After installation with \"torch\", \"transformer\", or \"all\" options, it's recommended to run the following CLI command to load the required models:\n\n```\n`crawl4ai-download-models `\n```\n\nThis is optional but will boost the performance and speed of the crawler. You only need to do this once after installation.\n\n## Playwright Installation Note for Ubuntu\n\nIf you encounter issues with Playwright installation on Ubuntu, you may need to install additional dependencies:\n\n```\n`sudo apt-get install -y \\ libwoff1 \\ libopus0 \\ libwebp7 \\ libwebpdemux2 \\ libenchant-2-2 \\ libgudev-1.0-0 \\ libsecret-1-0 \\ libhyphen0 \\ libgdk-pixbuf2.0-0 \\ libegl1 \\ libnotify4 \\ libxslt1.1 \\ libevent-2.1-7 \\ libgles2 \\ libxcomposite1 \\ libatk1.0-0 \\ libatk-bridge2.0-0 \\ libepoxy0 \\ libgtk-3-0 \\ libharfbuzz-icu0 \\ libgstreamer-gl1.0-0 \\ libgstreamer-plugins-bad1.0-0 \\ gstreamer1.0-plugins-good \\ gstreamer1.0-plugins-bad \\ libxt6 \\ libxaw7 \\ xvfb \\ fonts-noto-color-emoji \\ libfontconfig \\ libfreetype6 \\ xfonts-cyrillic \\ xfonts-scalable \\ fonts-liberation \\ fonts-ipafont-gothic \\ fonts-wqy-zenhei \\ fonts-tlwg-loma-otf \\ fonts-freefont-ttf `\n```\n\n## Option 2: Using Docker (Coming Soon)\n\nDocker support for Crawl4AI is currently in progress and will be available soon. This will allow you to run Crawl4AI in a containerized environment, ensuring consistency across different systems.\n\n## Option 3: Local Server Installation\n\nFor those who prefer to run Crawl4AI as a local server, instructions will be provided once the Docker implementation is complete.\n\n## Verifying Your Installation\n\nAfter installation, you can verify that Crawl4AI is working correctly by running a simple Python script:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler async def main(): async with AsyncWebCrawler(verbose=True) as crawler: result = await crawler.arun(url=\"https://www.example.com\") print(result.markdown[:500]) # Print first 500 characters if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\nThis script should successfully crawl the example website and print the first 500 characters of the extracted content.\n\n## Getting Help\n\nIf you encounter any issues during installation or usage, please check the [documentation](https://docs.crawl4ai.com/) or raise an issue on the [GitHub repository](https://github.com/unclecode/crawl4ai/issues).\n\nHappy crawling! 🕷️🤖\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](..)\n * [ Quick Start ](../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](..)\n * Setup & Installation\n * [Installation](../core/installation/)\n * [Docker Deployment](../core/docker-deployment/)\n * [Quick Start](../core/quickstart/)\n * Blog & Changelog\n * Blog Home\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../core/cli/)\n * [Simple Crawling](../core/simple-crawling/)\n * [Deep Crawling](../core/deep-crawling/)\n * [Crawler Result](../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../core/browser-crawler-config/)\n * [Markdown Generation](../core/markdown-generation/)\n * [Fit Markdown](../core/fit-markdown/)\n * [Page Interaction](../core/page-interaction/)\n * [Content Selection](../core/content-selection/)\n * [Cache Modes](../core/cache-modes/)\n * [Local Files & Raw HTML](../core/local-files/)\n * [Link & Media](../core/link-media/)\n * Advanced\n * [Overview](../advanced/advanced-features/)\n * [File Downloading](../advanced/file-downloading/)\n * [Lazy Loading](../advanced/lazy-loading/)\n * [Hooks & Auth](../advanced/hooks-auth/)\n * [Proxy & Security](../advanced/proxy-security/)\n * [Session Management](../advanced/session-management/)\n * [Multi-URL Crawling](../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../advanced/identity-based-crawling/)\n * [SSL Certificate](../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../extraction/no-llm-strategies/)\n * [LLM Strategies](../extraction/llm-strategies/)\n * [Clustering Strategies](../extraction/clustring-strategies/)\n * [Chunking](../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../api/async-webcrawler/)\n * [arun()](../api/arun/)\n * [arun_many()](../api/arun_many/)\n * [Browser, Crawler & LLM Config](../api/parameters/)\n * [CrawlResult](../api/crawl-result/)\n * [Strategies](../api/strategies/)\n\n\n\n * [Crawl4AI Blog](#crawl4ai-blog)\n * [Latest Release](#latest-release)\n * [License Change](#license-change)\n * [Project History](#project-history)\n * [Stay Updated](#stay-updated)\n\n\n\n# Crawl4AI Blog\n\nWelcome to the Crawl4AI blog! Here you'll find detailed release notes, technical insights, and updates about the project. Whether you're looking for the latest improvements or want to dive deep into web crawling techniques, this is the place.\n\n## Latest Release\n\n### [Crawl4AI v0.5.0: Deep Crawling, Scalability, and a New CLI!](releases/0.5.0/)\n\nMy dear friends and crawlers, there you go, this is the release of Crawl4AI v0.5.0! This release brings a wealth of new features, performance improvements, and a more streamlined developer experience. Here's a breakdown of what's new:\n\n**Major New Features:**\n\n * **Deep Crawling:** Explore entire websites with configurable strategies (BFS, DFS, Best-First). Define custom filters and URL scoring for targeted crawls.\n * **Memory-Adaptive Dispatcher:** Handle large-scale crawls with ease! Our new dispatcher dynamically adjusts concurrency based on available memory and includes built-in rate limiting.\n * **Multiple Crawler Strategies:** Choose between the full-featured Playwright browser-based crawler or a new, _much_ faster HTTP-only crawler for simpler tasks.\n * **Docker Deployment:** Deploy Crawl4AI as a scalable, self-contained service with built-in API endpoints and optional JWT authentication.\n * **Command-Line Interface (CLI):** Interact with Crawl4AI directly from your terminal. Crawl, configure, and extract data with simple commands.\n * **LLM Configuration (`LLMConfig`):** A new, unified way to configure LLM providers (OpenAI, Anthropic, Ollama, etc.) for extraction, filtering, and schema generation. Simplifies API key management and switching between models.\n\n\n\n**Minor Updates & Improvements:**\n\n * **LXML Scraping Mode:** Faster HTML parsing with `LXMLWebScrapingStrategy`.\n * **Proxy Rotation:** Added `ProxyRotationStrategy` with a `RoundRobinProxyStrategy` implementation.\n * **PDF Processing:** Extract text, images, and metadata from PDF files.\n * **URL Redirection Tracking:** Automatically follows and records redirects.\n * **Robots.txt Compliance:** Optionally respect website crawling rules.\n * **LLM-Powered Schema Generation:** Automatically create extraction schemas using an LLM.\n * **`LLMContentFilter`:** Generate high-quality, focused markdown using an LLM.\n * **Improved Error Handling & Stability:** Numerous bug fixes and performance enhancements.\n * **Enhanced Documentation:** Updated guides and examples.\n\n\n\n**Breaking Changes & Migration:**\n\nThis release includes several breaking changes to improve the library's structure and consistency. Here's what you need to know:\n\n * **`arun_many()` Behavior:** Now uses the `MemoryAdaptiveDispatcher` by default. The return type depends on the `stream` parameter in `CrawlerRunConfig`. Adjust code that relied on unbounded concurrency.\n * **`max_depth` Location:** Moved to `CrawlerRunConfig` and now controls _crawl depth_.\n * **Deep Crawling Imports:** Import `DeepCrawlStrategy` and related classes from `crawl4ai.deep_crawling`.\n * **`BrowserContext` API:** Updated; the old `get_context` method is deprecated.\n * **Optional Model Fields:** Many data model fields are now optional. Handle potential `None` values.\n * **`ScrapingMode` Enum:** Replaced with strategy pattern (`WebScrapingStrategy`, `LXMLWebScrapingStrategy`).\n * **`content_filter` Parameter:** Removed from `CrawlerRunConfig`. Use extraction strategies or markdown generators with filters.\n * **Removed Functionality:** The synchronous `WebCrawler`, the old CLI, and docs management tools have been removed.\n * **Docker:** Significant changes to deployment. See the [Docker documentation](../deploy/docker/README.md).\n * **`ssl_certificate.json`:** This file has been removed.\n * **Config** : FastFilterChain has been replaced with FilterChain\n * **Deep-Crawl** : DeepCrawlStrategy.arun now returns Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]\n * **Proxy** : Removed synchronous WebCrawler support and related rate limiting configurations\n * **LLM Parameters:** Use the new `LLMConfig` object instead of passing `provider`, `api_token`, `base_url`, and `api_base` directly to `LLMExtractionStrategy` and `LLMContentFilter`.\n\n\n\n**In short:** Update imports, adjust `arun_many()` usage, check for optional fields, and review the Docker deployment guide.\n\n## License Change\n\nCrawl4AI v0.5.0 updates the license to Apache 2.0 _with a required attribution clause_. This means you are free to use, modify, and distribute Crawl4AI (even commercially), but you _must_ clearly attribute the project in any public use or distribution. See the updated `LICENSE` file for the full legal text and specific requirements.\n\n**Get Started:**\n\n * **Installation:** `pip install \"crawl4ai[all]\"` (or use the Docker image)\n * **Documentation:** \n * **GitHub:** \n\n\n\nI'm very excited to see what you build with Crawl4AI v0.5.0!\n\n### [0.4.2 - Configurable Crawlers, Session Management, and Smarter Screenshots](releases/0.4.2/)\n\n_December 12, 2024_\n\nThe 0.4.2 update brings massive improvements to configuration, making crawlers and browsers easier to manage with dedicated objects. You can now import/export local storage for seamless session management. Plus, long-page screenshots are faster and cleaner, and full-page PDF exports are now possible. Check out all the new features to make your crawling experience even smoother.\n\n[Read full release notes →](releases/0.4.2/)\n\n### [0.4.1 - Smarter Crawling with Lazy-Load Handling, Text-Only Mode, and More](releases/0.4.1/)\n\n_December 8, 2024_\n\nThis release brings major improvements to handling lazy-loaded images, a blazing-fast Text-Only Mode, full-page scanning for infinite scrolls, dynamic viewport adjustments, and session reuse for efficient crawling. If you're looking to improve speed, reliability, or handle dynamic content with ease, this update has you covered.\n\n[Read full release notes →](releases/0.4.1/)\n\n### [0.4.0 - Major Content Filtering Update](releases/0.4.0/)\n\n_December 1, 2024_\n\nIntroduced significant improvements to content filtering, multi-threaded environment handling, and user-agent generation. This release features the new PruningContentFilter, enhanced thread safety, and improved test coverage.\n\n[Read full release notes →](releases/0.4.0/)\n\n## Project History\n\nCurious about how Crawl4AI has evolved? Check out our [complete changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md) for a detailed history of all versions and updates.\n\n## Stay Updated\n\n * Star us on [GitHub](https://github.com/unclecode/crawl4ai)\n * Follow [@unclecode](https://twitter.com/unclecode) on Twitter\n * Join our community discussions on GitHub\n\n\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [Introducing Event Streams and Interactive Hooks in Crawl4AI](#introducing-event-streams-and-interactive-hooks-in-crawl4ai)\n\n\n\n## Introducing Event Streams and Interactive Hooks in Crawl4AI\n\n![event-driven-crawl](https://res.cloudinary.com/kidocode/image/upload/t_400x400/v1734344008/15bb8bbb-83ac-43ac-962d-3feb3e0c3bbf_2_tjmr4n.webp)\n\nIn the near future, I’m planning to enhance Crawl4AI’s capabilities by introducing an event stream mechanism that will give clients deeper, real-time insights into the crawling process. Today, hooks are a powerful feature at the code level—they let developers define custom logic at key points in the crawl. However, when using Crawl4AI as a service (e.g., through a Dockerized API), there isn’t an easy way to interact with these hooks at runtime.\n\n**What’s Changing?**\n\nI’m working on a solution that will allow the crawler to emit a continuous stream of events, updating clients on the current crawling stage, encountered pages, and any decision points. This event stream could be exposed over a standardized protocol like Server-Sent Events (SSE) or WebSockets, enabling clients to “subscribe” and listen as the crawler works.\n\n**Interactivity Through Process IDs**\n\nA key part of this new design is the concept of a unique process ID for each crawl session. Imagine you’re listening to an event stream that informs you: - The crawler just hit a certain page - It triggered a hook and is now pausing for instructions \n\nWith the event stream in place, you can send a follow-up request back to the server—referencing the unique process ID—to provide extra data, instructions, or parameters. This might include selecting which links to follow next, adjusting extraction strategies, or providing authentication tokens for a protected API. Once the crawler receives these instructions, it resumes execution with the updated context.\n\n```\n`sequenceDiagram participant Client participant Server participant Crawler Client->>Server: Start crawl request Server->>Crawler: Initiate crawl with Process ID Crawler-->>Server: Event: Page hit Server-->>Client: Stream: Page hit event Client->>Server: Instruction for Process ID Server->>Crawler: Update crawl with new instructions Crawler-->>Server: Event: Crawl completed Server-->>Client: Stream: Crawl completed `\n```\n\n**Benefits for Developers and Users**\n\n1. **Fine-Grained Control** : Instead of predefining all logic upfront, you can dynamically guide the crawler in response to actual data and conditions encountered mid-crawl. 2. **Real-Time Insights** : Monitor progress, errors, or network bottlenecks as they happen, without waiting for the entire crawl to finish. 3. **Enhanced Collaboration** : Different team members or automated systems can watch the same crawl events and provide input, making the crawling process more adaptive and intelligent.\n\n**Next Steps**\n\nI’m currently exploring the best APIs, technologies, and patterns to make this vision a reality. My goal is to deliver a seamless developer experience—one that integrates with existing Crawl4AI workflows while offering new flexibility and power.\n\nStay tuned for more updates as I continue building this feature out. In the meantime, I’d love to hear any feedback or suggestions you might have to help shape this interactive, event-driven future of web crawling with Crawl4AI.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [Release Summary for Version 0.4.0 (December 1, 2024)](#release-summary-for-version-040-december-1-2024)\n * [Overview](#overview)\n * [Major Features and Enhancements](#major-features-and-enhancements)\n * [Revised Change Logs for Version 0.4.0](#revised-change-logs-for-version-040)\n * [Experimental Features](#experimental-features)\n * [Conclusion](#conclusion)\n\n\n\n# Release Summary for Version 0.4.0 (December 1, 2024)\n\n## Overview\n\nThe 0.4.0 release introduces significant improvements to content filtering, multi-threaded environment handling, user-agent generation, and test coverage. Key highlights include the introduction of the PruningContentFilter, designed to automatically identify and extract the most valuable parts of an HTML document, as well as enhancements to the BM25ContentFilter to extend its versatility and effectiveness.\n\n## Major Features and Enhancements\n\n### 1. PruningContentFilter\n\n * Introduced a new unsupervised content filtering strategy that scores and prunes less relevant nodes in an HTML document based on metrics like text and link density.\n * Focuses on retaining the most valuable parts of the content, making it highly effective for extracting relevant information from complex web pages.\n * Fully documented with updated README and expanded user guides.\n\n\n\n### 2. User-Agent Generator\n\n * Added a user-agent generator utility that resolves compatibility issues and supports customizable user-agent strings.\n * By default, the generator randomizes user agents for each request, adding diversity, but users can customize it for tailored scenarios.\n\n\n\n### 3. Enhanced Thread Safety\n\n * Improved handling of multi-threaded environments by adding better thread locks for parallel processing, ensuring consistency and stability when running multiple threads.\n\n\n\n### 4. Extended Content Filtering Strategies\n\n * Users now have access to both the PruningContentFilter for unsupervised extraction and the BM25ContentFilter for supervised filtering based on user queries.\n * Enhanced BM25ContentFilter with improved capabilities to process page titles, meta tags, and descriptions, allowing for more effective classification and clustering of text chunks.\n\n\n\n### 5. Documentation Updates\n\n * Updated examples and tutorials to promote the use of the PruningContentFilter alongside the BM25ContentFilter, providing clear instructions for selecting the appropriate filter for each use case.\n\n\n\n### 6. Unit Test Enhancements\n\n * Added unit tests for PruningContentFilter to ensure accuracy and reliability.\n * Enhanced BM25ContentFilter tests to cover additional edge cases and performance metrics, particularly for malformed HTML inputs.\n\n\n\n## Revised Change Logs for Version 0.4.0\n\n### PruningContentFilter (Dec 01, 2024)\n\n * Introduced the PruningContentFilter to optimize content extraction by pruning less relevant HTML nodes.\n * **Affected Files:**\n * **crawl4ai/content_filter_strategy.py** : Added a scoring-based pruning algorithm.\n * **README.md** : Updated to include PruningContentFilter usage.\n * **docs/md_v2/basic/content_filtering.md** : Expanded user documentation, detailing the use and benefits of PruningContentFilter.\n\n\n\n### Unit Tests for PruningContentFilter (Dec 01, 2024)\n\n * Added comprehensive unit tests for PruningContentFilter to ensure correctness and efficiency.\n * **Affected Files:**\n * **tests/async/test_content_filter_prune.py** : Created tests covering different pruning scenarios to ensure stability and correctness.\n\n\n\n### Enhanced BM25ContentFilter Tests (Dec 01, 2024)\n\n * Expanded tests to cover additional extraction scenarios and performance metrics, improving robustness.\n * **Affected Files:**\n * **tests/async/test_content_filter_bm25.py** : Added tests for edge cases, including malformed HTML inputs.\n\n\n\n### Documentation and Example Updates (Dec 01, 2024)\n\n * Revised examples to illustrate the use of PruningContentFilter alongside existing content filtering methods.\n * **Affected Files:**\n * **docs/examples/quickstart_async.py** : Enhanced example clarity and usability for new users.\n\n\n\n## Experimental Features\n\n * The PruningContentFilter is still under experimental development, and we continue to gather feedback for further refinements.\n\n\n\n## Conclusion\n\nThis release significantly enhances the content extraction capabilities of Crawl4ai with the introduction of the PruningContentFilter, improved supervised filtering with BM25ContentFilter, and robust multi-threaded handling. Additionally, the user-agent generator provides much-needed versatility, resolving compatibility issues faced by many users.\n\nUsers are encouraged to experiment with the new content filtering methods to determine which best suits their needs.\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [Release Summary for Version 0.4.1 (December 8, 2024): Major Efficiency Boosts with New Features!](#release-summary-for-version-041-december-8-2024-major-efficiency-boosts-with-new-features)\n * [Handling Lazy Loading Better (Images Included)](#handling-lazy-loading-better-images-included)\n * [Text-Only Mode (Fast, Lightweight Crawling)](#text-only-mode-fast-lightweight-crawling)\n * [Adjusting the Viewport Dynamically](#adjusting-the-viewport-dynamically)\n * [Simulating Full-Page Scrolling](#simulating-full-page-scrolling)\n * [Reusing Browser Sessions (Save Time on Setup)](#reusing-browser-sessions-save-time-on-setup)\n * [Other Updates](#other-updates)\n * [How to Get the Update](#how-to-get-the-update)\n\n\n\n# Release Summary for Version 0.4.1 (December 8, 2024): Major Efficiency Boosts with New Features!\n\n_This post was generated with the help of ChatGPT, take everything with a grain of salt. 🧂_\n\nHi everyone,\n\nI just finished putting together version 0.4.1 of Crawl4AI, and there are a few changes in here that I think you’ll find really helpful. I’ll explain what’s new, why it matters, and exactly how you can use these features (with the code to back it up). Let’s get into it.\n\n### Handling Lazy Loading Better (Images Included)\n\nOne thing that always bugged me with crawlers is how often they miss lazy-loaded content, especially images. In this version, I made sure Crawl4AI **waits for all images to load** before moving forward. This is useful because many modern websites only load images when they’re in the viewport or after some JavaScript executes.\n\nHere’s how to enable it:\n\n```\n`await crawler.crawl( url=\"https://example.com\", wait_for_images=True # Add this argument to ensure images are fully loaded ) `\n```\n\nWhat this does is: 1. Waits for the page to reach a \"network idle\" state. 2. Ensures all images on the page have been completely loaded.\n\nThis single change handles the majority of lazy-loading cases you’re likely to encounter.\n\n### Text-Only Mode (Fast, Lightweight Crawling)\n\nSometimes, you don’t need to download images or process JavaScript at all. For example, if you’re crawling to extract text data, you can enable **text-only mode** to speed things up. By disabling images, JavaScript, and other heavy resources, this mode makes crawling **3-4 times faster** in most cases.\n\nHere’s how to turn it on:\n\n```\n`crawler = AsyncPlaywrightCrawlerStrategy( text_mode=True # Set this to True to enable text-only crawling ) `\n```\n\nWhen `text_mode=True`, the crawler automatically: - Disables GPU processing. - Blocks image and JavaScript resources. - Reduces the viewport size to 800x600 (you can override this with `viewport_width` and `viewport_height`).\n\nIf you need to crawl thousands of pages where you only care about text, this mode will save you a ton of time and resources.\n\n### Adjusting the Viewport Dynamically\n\nAnother useful addition is the ability to **dynamically adjust the viewport size** to match the content on the page. This is particularly helpful when you’re working with responsive layouts or want to ensure all parts of the page load properly.\n\nHere’s how it works: 1. The crawler calculates the page’s width and height after it loads. 2. It adjusts the viewport to fit the content dimensions. 3. (Optional) It uses Chrome DevTools Protocol (CDP) to simulate zooming out so everything fits in the viewport.\n\nTo enable this, use:\n\n```\n`await crawler.crawl( url=\"https://example.com\", adjust_viewport_to_content=True # Dynamically adjusts the viewport ) `\n```\n\nThis approach makes sure the entire page gets loaded into the viewport, especially for layouts that load content based on visibility.\n\n### Simulating Full-Page Scrolling\n\nSome websites load data dynamically as you scroll down the page. To handle these cases, I added support for **full-page scanning**. It simulates scrolling to the bottom of the page, checking for new content, and capturing it all.\n\nHere’s an example:\n\n```\n`await crawler.crawl( url=\"https://example.com\", scan_full_page=True, # Enables scrolling scroll_delay=0.2 # Waits 200ms between scrolls (optional) ) `\n```\n\nWhat happens here: 1. The crawler scrolls down in increments, waiting for content to load after each scroll. 2. It stops when no new content appears (i.e., dynamic elements stop loading). 3. It scrolls back to the top before finishing (if necessary).\n\nIf you’ve ever had to deal with infinite scroll pages, this is going to save you a lot of headaches.\n\n### Reusing Browser Sessions (Save Time on Setup)\n\nBy default, every time you crawl a page, a new browser context (or tab) is created. That’s fine for small crawls, but if you’re working on a large dataset, it’s more efficient to reuse the same session.\n\nI added a method called `create_session` for this:\n\n```\n`session_id = await crawler.create_session() # Use the same session for multiple crawls await crawler.crawl( url=\"https://example.com/page1\", session_id=session_id # Reuse the session ) await crawler.crawl( url=\"https://example.com/page2\", session_id=session_id ) `\n```\n\nThis avoids creating a new tab for every page, speeding up the crawl and reducing memory usage.\n\n### Other Updates\n\nHere are a few smaller updates I’ve made: - **Light Mode** : Use `light_mode=True` to disable background processes, extensions, and other unnecessary features, making the browser more efficient. - **Logging** : Improved logs to make debugging easier. - **Defaults** : Added sensible defaults for things like `delay_before_return_html` (now set to 0.1 seconds).\n\n### How to Get the Update\n\nYou can install or upgrade to version `0.4.1` like this:\n\n```\n`pip install crawl4ai --upgrade `\n```\n\nAs always, I’d love to hear your thoughts. If there’s something you think could be improved or if you have suggestions for future versions, let me know!\n\nEnjoy the new features, and happy crawling! 🕷️\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [🚀 Crawl4AI 0.4.2 Update: Smarter Crawling Just Got Easier (Dec 12, 2024)](#crawl4ai-042-update-smarter-crawling-just-got-easier-dec-12-2024)\n * [Hey Developers,](#hey-developers)\n * [🔧 Configurable Browser and Crawler Behavior](#configurable-browser-and-crawler-behavior)\n * [🔐 Streamlined Session Management](#streamlined-session-management)\n * [🔢 Handling Large Pages: Supercharged Screenshots and PDF Conversion](#handling-large-pages-supercharged-screenshots-and-pdf-conversion)\n * [🔧 Other Cool Stuff](#other-cool-stuff)\n * [📊 Performance Boosts and Dev-friendly Updates](#performance-boosts-and-dev-friendly-updates)\n * [🔠 Use Cases You’ll Love](#use-cases-youll-love)\n * [Let’s Get Crawling](#lets-get-crawling)\n\n\n\n## 🚀 Crawl4AI 0.4.2 Update: Smarter Crawling Just Got Easier (Dec 12, 2024)\n\n### Hey Developers,\n\nI’m excited to share Crawl4AI 0.4.2—a major upgrade that makes crawling smarter, faster, and a whole lot more intuitive. I’ve packed in a bunch of new features to simplify your workflows and improve your experience. Let’s cut to the chase!\n\n### 🔧 **Configurable Browser and Crawler Behavior**\n\nYou’ve asked for better control over how browsers and crawlers are configured, and now you’ve got it. With the new `BrowserConfig` and `CrawlerRunConfig` objects, you can set up your browser and crawling behavior exactly how you want. No more cluttering `arun` with a dozen arguments—just pass in your configs and go.\n\n**Example:**\n\n```\n`from crawl4ai import BrowserConfig, CrawlerRunConfig, AsyncWebCrawler browser_config = BrowserConfig(headless=True, viewport_width=1920, viewport_height=1080) crawler_config = CrawlerRunConfig(cache_mode=\"BYPASS\") async with AsyncWebCrawler(config=browser_config) as crawler: result = await crawler.arun(url=\"https://example.com\", config=crawler_config) print(result.markdown[:500]) `\n```\n\nThis setup is a game-changer for scalability, keeping your code clean and flexible as we add more parameters in the future.\n\nRemember: If you like to use the old way, you can still pass arguments directly to `arun` as before, no worries!\n\n### 🔐 **Streamlined Session Management**\n\nHere’s the big one: You can now pass local storage and cookies directly. Whether it’s setting values programmatically or importing a saved JSON state, managing sessions has never been easier. This is a must-have for authenticated crawls—just export your storage state once and reuse it effortlessly across runs.\n\n**Example:** 1. Open a browser, log in manually, and export the storage state. 2. Import the JSON file for seamless authenticated crawling:\n\n```\n`result = await crawler.arun( url=\"https://example.com/protected\", storage_state=\"my_storage_state.json\" ) `\n```\n\n### 🔢 **Handling Large Pages: Supercharged Screenshots and PDF Conversion**\n\nTwo big upgrades here:\n\n * **Blazing-fast long-page screenshots** : Turn extremely long web pages into clean, high-quality screenshots—without breaking a sweat. It’s optimized to handle large content without lag.\n\n * **Full-page PDF exports** : Now, you can also convert any page into a PDF with all the details intact. Perfect for archiving or sharing complex layouts.\n\n\n\n\n### 🔧 **Other Cool Stuff**\n\n * **Anti-bot enhancements** : Magic mode now handles overlays, user simulation, and anti-detection features like a pro.\n * **JavaScript execution** : Execute custom JS snippets to handle dynamic content. No more wrestling with endless page interactions.\n\n\n\n### 📊 **Performance Boosts and Dev-friendly Updates**\n\n * Faster rendering and viewport adjustments for better performance.\n * Improved cookie and local storage handling for seamless authentication.\n * Better debugging with detailed logs and actionable error messages.\n\n\n\n### 🔠 **Use Cases You’ll Love**\n\n1. **Authenticated Crawls** : Login once, export your storage state, and reuse it across multiple requests without the headache. 2. **Long-page Screenshots** : Perfect for blogs, e-commerce pages, or any endless-scroll website. 3. **PDF Export** : Create professional-looking page PDFs in seconds.\n\n### Let’s Get Crawling\n\nCrawl4AI 0.4.2 is ready for you to download and try. I’m always looking for ways to improve, so don’t hold back—share your thoughts and feedback.\n\nHappy Crawling! 🚀\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [Crawl4AI v0.5.0 Release Notes](#crawl4ai-v050-release-notes)\n * [Key Features](#key-features)\n * [Other Changes and Improvements](#other-changes-and-improvements)\n * [Breaking Changes Summary](#breaking-changes-summary)\n * [Migration Guide](#migration-guide)\n\n\n\n# Crawl4AI v0.5.0 Release Notes\n\n**Release Theme: Power, Flexibility, and Scalability**\n\nCrawl4AI v0.5.0 is a major release focused on significantly enhancing the library's power, flexibility, and scalability. Key improvements include a new **deep crawling** system, a **memory-adaptive dispatcher** for handling large-scale crawls, **multiple crawling strategies** (including a fast HTTP-only crawler), **Docker** deployment options, and a powerful **command-line interface (CLI)**. This release also includes numerous bug fixes, performance optimizations, and documentation updates.\n\n**Important Note:** This release contains several **breaking changes**. Please review the \"Breaking Changes\" section carefully and update your code accordingly.\n\n## Key Features\n\n### 1. Deep Crawling\n\nCrawl4AI now supports deep crawling, allowing you to explore websites beyond the initial URLs. This is controlled by the `deep_crawl_strategy` parameter in `CrawlerRunConfig`. Several strategies are available:\n\n * **`BFSDeepCrawlStrategy` (Breadth-First Search):** Explores the website level by level. (Default)\n * **`DFSDeepCrawlStrategy` (Depth-First Search):** Explores each branch as deeply as possible before backtracking.\n * **`BestFirstCrawlingStrategy`:** Uses a scoring function to prioritize which URLs to crawl next.\n\n\n\n```\n`import time from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BFSDeepCrawlStrategy from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy from crawl4ai.deep_crawling import DomainFilter, ContentTypeFilter, FilterChain, URLPatternFilter, KeywordRelevanceScorer, BestFirstCrawlingStrategy import asyncio # Create a filter chain to filter urls based on patterns, domains and content type filter_chain = FilterChain( [ DomainFilter( allowed_domains=[\"docs.crawl4ai.com\"], blocked_domains=[\"old.docs.crawl4ai.com\"], ), URLPatternFilter(patterns=[\"*core*\", \"*advanced*\"],), ContentTypeFilter(allowed_types=[\"text/html\"]), ] ) # Create a keyword scorer that prioritises the pages with certain keywords first keyword_scorer = KeywordRelevanceScorer( keywords=[\"crawl\", \"example\", \"async\", \"configuration\"], weight=0.7 ) # Set up the configuration deep_crawl_config = CrawlerRunConfig( deep_crawl_strategy=BestFirstCrawlingStrategy( max_depth=2, include_external=False, filter_chain=filter_chain, url_scorer=keyword_scorer, ), scraping_strategy=LXMLWebScrapingStrategy(), stream=True, verbose=True, ) async def main(): async with AsyncWebCrawler() as crawler: start_time = time.perf_counter() results = [] async for result in await crawler.arun(url=\"https://docs.crawl4ai.com\", config=deep_crawl_config): print(f\"Crawled: {result.url} (Depth: {result.metadata['depth']}), score: {result.metadata['score']:.2f}\") results.append(result) duration = time.perf_counter() - start_time print(f\"\\n✅ Crawled {len(results)} high-value pages in {duration:.2f} seconds\") asyncio.run(main()) `\n```\n\n**Breaking Change:** The `max_depth` parameter is now part of `CrawlerRunConfig` and controls the _depth_ of the crawl, not the number of concurrent crawls. The `arun()` and `arun_many()` methods are now decorated to handle deep crawling strategies. Imports for deep crawling strategies have changed. See the [Deep Crawling documentation](../../../core/deep-crawling/) for more details.\n\n### 2. Memory-Adaptive Dispatcher\n\nThe new `MemoryAdaptiveDispatcher` dynamically adjusts concurrency based on available system memory and includes built-in rate limiting. This prevents out-of-memory errors and avoids overwhelming target websites.\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MemoryAdaptiveDispatcher import asyncio # Configure the dispatcher (optional, defaults are used if not provided) dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=80.0, # Pause if memory usage exceeds 80% check_interval=0.5, # Check memory every 0.5 seconds ) async def batch_mode(): async with AsyncWebCrawler() as crawler: results = await crawler.arun_many( urls=[\"https://docs.crawl4ai.com\", \"https://github.com/unclecode/crawl4ai\"], config=CrawlerRunConfig(stream=False), # Batch mode dispatcher=dispatcher, ) for result in results: print(f\"Crawled: {result.url} with status code: {result.status_code}\") async def stream_mode(): async with AsyncWebCrawler() as crawler: # OR, for streaming: async for result in await crawler.arun_many( urls=[\"https://docs.crawl4ai.com\", \"https://github.com/unclecode/crawl4ai\"], config=CrawlerRunConfig(stream=True), dispatcher=dispatcher, ): print(f\"Crawled: {result.url} with status code: {result.status_code}\") print(\"Dispatcher in batch mode:\") asyncio.run(batch_mode()) print(\"-\" * 50) print(\"Dispatcher in stream mode:\") asyncio.run(stream_mode()) `\n```\n\n**Breaking Change:** `AsyncWebCrawler.arun_many()` now uses `MemoryAdaptiveDispatcher` by default. Existing code that relied on unbounded concurrency may require adjustments.\n\n### 3. Multiple Crawling Strategies (Playwright and HTTP)\n\nCrawl4AI now offers two crawling strategies:\n\n * **`AsyncPlaywrightCrawlerStrategy` (Default):** Uses Playwright for browser-based crawling, supporting JavaScript rendering and complex interactions.\n * **`AsyncHTTPCrawlerStrategy`:** A lightweight, fast, and memory-efficient HTTP-only crawler. Ideal for simple scraping tasks where browser rendering is unnecessary.\n\n\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, HTTPCrawlerConfig from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy import asyncio # Use the HTTP crawler strategy http_crawler_config = HTTPCrawlerConfig( method=\"GET\", headers={\"User-Agent\": \"MyCustomBot/1.0\"}, follow_redirects=True, verify_ssl=True ) async def main(): async with AsyncWebCrawler(crawler_strategy=AsyncHTTPCrawlerStrategy(browser_config =http_crawler_config)) as crawler: result = await crawler.arun(\"https://example.com\") print(f\"Status code: {result.status_code}\") print(f\"Content length: {len(result.html)}\") asyncio.run(main()) `\n```\n\n### 4. Docker Deployment\n\nCrawl4AI can now be easily deployed as a Docker container, providing a consistent and isolated environment. The Docker image includes a FastAPI server with both streaming and non-streaming endpoints.\n\n```\n`# Build the image (from the project root) docker build -t crawl4ai . # Run the container docker run -d -p 8000:8000 --name crawl4ai crawl4ai `\n```\n\n**API Endpoints:**\n\n * `/crawl` (POST): Non-streaming crawl.\n * `/crawl/stream` (POST): Streaming crawl (NDJSON).\n * `/health` (GET): Health check.\n * `/schema` (GET): Returns configuration schemas.\n * `/md/{url}` (GET): Returns markdown content of the URL.\n * `/llm/{url}` (GET): Returns LLM extracted content.\n * `/token` (POST): Get JWT token\n\n\n\n**Breaking Changes:**\n\n * Docker deployment now requires a `.llm.env` file for API keys.\n * Docker deployment now requires Redis and a new `config.yml` structure.\n * Server startup now uses `supervisord` instead of direct process management.\n * Docker server now requires authentication by default (JWT tokens).\n\n\n\nSee the [Docker deployment documentation](../../../core/docker-deployment/) for detailed instructions.\n\n### 5. Command-Line Interface (CLI)\n\nA new CLI (`crwl`) provides convenient access to Crawl4AI's functionality from the terminal.\n\n```\n`# Basic crawl crwl https://example.com # Get markdown output crwl https://example.com -o markdown # Use a configuration file crwl https://example.com -B browser.yml -C crawler.yml # Use LLM-based extraction crwl https://example.com -e extract.yml -s schema.json # Ask a question about the crawled content crwl https://example.com -q \"What is the main topic?\" # See usage examples crwl --example `\n```\n\nSee the [CLI documentation](../docs/md_v2/core/cli.md) for more details.\n\n### 6. LXML Scraping Mode\n\nAdded `LXMLWebScrapingStrategy` for faster HTML parsing using the `lxml` library. This can significantly improve scraping performance, especially for large or complex pages. Set `scraping_strategy=LXMLWebScrapingStrategy()` in your `CrawlerRunConfig`.\n\n**Breaking Change:** The `ScrapingMode` enum has been replaced with a strategy pattern. Use `WebScrapingStrategy` (default) or `LXMLWebScrapingStrategy`.\n\n### 7. Proxy Rotation\n\nAdded `ProxyRotationStrategy` abstract base class with `RoundRobinProxyStrategy` concrete implementation.\n\n```\n`import re from crawl4ai import ( AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, RoundRobinProxyStrategy, ) import asyncio from crawl4ai.proxy_strategy import ProxyConfig async def main(): # Load proxies and create rotation strategy proxies = ProxyConfig.from_env() #eg: export PROXIES=\"ip1:port1:username1:password1,ip2:port2:username2:password2\" if not proxies: print(\"No proxies found in environment. Set PROXIES env variable!\") return proxy_strategy = RoundRobinProxyStrategy(proxies) # Create configs browser_config = BrowserConfig(headless=True, verbose=False) run_config = CrawlerRunConfig( cache_mode=CacheMode.BYPASS, proxy_rotation_strategy=proxy_strategy ) async with AsyncWebCrawler(config=browser_config) as crawler: urls = [\"https://httpbin.org/ip\"] * (len(proxies) * 2) # Test each proxy twice print(\"\\n📈 Initializing crawler with proxy rotation...\") async with AsyncWebCrawler(config=browser_config) as crawler: print(\"\\n🚀 Starting batch crawl with proxy rotation...\") results = await crawler.arun_many( urls=urls, config=run_config ) for result in results: if result.success: ip_match = re.search(r'(?:[0-9]{1,3}\\.){3}[0-9]{1,3}', result.html) current_proxy = run_config.proxy_config if run_config.proxy_config else None if current_proxy and ip_match: print(f\"URL {result.url}\") print(f\"Proxy {current_proxy.server} -> Response IP: {ip_match.group(0)}\") verified = ip_match.group(0) == current_proxy.ip if verified: print(f\"✅ Proxy working! IP matches: {current_proxy.ip}\") else: print(\"❌ Proxy failed or IP mismatch!\") print(\"---\") asyncio.run(main()) `\n```\n\n## Other Changes and Improvements\n\n * **Added:`LLMContentFilter` for intelligent markdown generation.** This new filter uses an LLM to create more focused and relevant markdown output.\n\n\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, DefaultMarkdownGenerator from crawl4ai.content_filter_strategy import LLMContentFilter from crawl4ai import LLMConfig import asyncio llm_config = LLMConfig(provider=\"gemini/gemini-1.5-pro\", api_token=\"env:GEMINI_API_KEY\") markdown_generator = DefaultMarkdownGenerator( content_filter=LLMContentFilter(llm_config=llm_config, instruction=\"Extract key concepts and summaries\") ) config = CrawlerRunConfig(markdown_generator=markdown_generator) async def main(): async with AsyncWebCrawler() as crawler: result = await crawler.arun(\"https://docs.crawl4ai.com\", config=config) print(result.markdown.fit_markdown) asyncio.run(main()) `\n```\n\n * **Added: URL redirection tracking.** The crawler now automatically follows HTTP redirects (301, 302, 307, 308) and records the final URL in the `redirected_url` field of the `CrawlResult` object. No code changes are required to enable this; it's automatic.\n\n * **Added: LLM-powered schema generation utility.** A new `generate_schema` method has been added to `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy`. This greatly simplifies creating extraction schemas.\n\n\n\n\n```\n`from crawl4ai.extraction_strategy import JsonCssExtractionStrategy from crawl4ai import LLMConfig llm_config = LLMConfig(provider=\"gemini/gemini-1.5-pro\", api_token=\"env:GEMINI_API_KEY\") schema = JsonCssExtractionStrategy.generate_schema( html=\"

Product Name

$99
\", llm_config = llm_config, query=\"Extract product name and price\" ) print(schema) `\n```\n\nExpected Output (may vary slightly due to LLM) \n\n```\n`{ \"name\": \"ProductExtractor\", \"baseSelector\": \"div.product\", \"fields\": [ {\"name\": \"name\", \"selector\": \"h2\", \"type\": \"text\"}, {\"name\": \"price\", \"selector\": \".price\", \"type\": \"text\"} ] } `\n```\n\n * **Added: robots.txt compliance support.** The crawler can now respect `robots.txt` rules. Enable this by setting `check_robots_txt=True` in `CrawlerRunConfig`.\n\n\n\n```\n`config = CrawlerRunConfig(check_robots_txt=True) `\n```\n\n * **Added: PDF processing capabilities.** Crawl4AI can now extract text, images, and metadata from PDF files (both local and remote). This uses a new `PDFCrawlerStrategy` and `PDFContentScrapingStrategy`.\n\n\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy import asyncio async def main(): async with AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) as crawler: result = await crawler.arun( \"https://arxiv.org/pdf/2310.06825.pdf\", config=CrawlerRunConfig( scraping_strategy=PDFContentScrapingStrategy() ) ) print(result.markdown) # Access extracted text print(result.metadata) # Access PDF metadata (title, author, etc.) asyncio.run(main()) `\n```\n\n * **Added: Support for frozenset serialization.** Improves configuration serialization, especially for sets of allowed/blocked domains. No code changes required.\n\n * **Added: New`LLMConfig` parameter.** This new parameter can be passed for extraction, filtering, and schema generation tasks. It simplifies passing provider strings, API tokens, and base URLs across all sections where LLM configuration is necessary. It also enables reuse and allows for quick experimentation between different LLM configurations.\n\n\n\n\n```\n`from crawl4ai import LLMConfig from crawl4ai.extraction_strategy import LLMExtractionStrategy from crawl4ai import AsyncWebCrawler, CrawlerRunConfig # Example of using LLMConfig with LLMExtractionStrategy llm_config = LLMConfig(provider=\"openai/gpt-4o\", api_token=\"YOUR_API_KEY\") strategy = LLMExtractionStrategy(llm_config=llm_config, schema=...) # Example usage within a crawler async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://example.com\", config=CrawlerRunConfig(extraction_strategy=strategy) ) `\n```\n\n**Breaking Change:** Removed old parameters like `provider`, `api_token`, `base_url`, and `api_base` from `LLMExtractionStrategy` and `LLMContentFilter`. Users should migrate to using the `LLMConfig` object. \n\n * **Changed: Improved browser context management and added shared data support. (Breaking Change:** `BrowserContext` API updated). Browser contexts are now managed more efficiently, reducing resource usage. A new `shared_data` dictionary is available in the `BrowserContext` to allow passing data between different stages of the crawling process. **Breaking Change:** The `BrowserContext` API has changed, and the old `get_context` method is deprecated.\n\n * **Changed:** Renamed `final_url` to `redirected_url` in `CrawledURL`. This improves consistency and clarity. Update any code referencing the old field name.\n\n * **Changed:** Improved type hints and removed unused files. This is an internal improvement and should not require code changes.\n\n * **Changed:** Reorganized deep crawling functionality into dedicated module. (**Breaking Change:** Import paths for `DeepCrawlStrategy` and related classes have changed). This improves code organization. Update imports to use the new `crawl4ai.deep_crawling` module.\n\n * **Changed:** Improved HTML handling and cleanup codebase. (**Breaking Change:** Removed `ssl_certificate.json` file). This removes an unused file. If you were relying on this file for custom certificate validation, you'll need to implement an alternative approach.\n\n * **Changed:** Enhanced serialization and config handling. (**Breaking Change:** `FastFilterChain` has been replaced with `FilterChain`). This change simplifies config and improves the serialization.\n\n * **Added:** Modified the license to Apache 2.0 _with a required attribution clause_. See the `LICENSE` file for details. All users must now clearly attribute the Crawl4AI project when using, distributing, or creating derivative works.\n\n * **Fixed:** Prevent memory leaks by ensuring proper closure of Playwright pages. No code changes required.\n\n * **Fixed:** Make model fields optional with default values (**Breaking Change:** Code relying on all fields being present may need adjustment). Fields in data models (like `CrawledURL`) are now optional, with default values (usually `None`). Update code to handle potential `None` values.\n\n * **Fixed:** Adjust memory threshold and fix dispatcher initialization. This is an internal bug fix; no code changes are required.\n\n * **Fixed:** Ensure proper exit after running doctor command. No code changes are required.\n\n * **Fixed:** JsonCss selector and crawler improvements.\n * **Fixed:** Not working long page screenshot (#403)\n * **Documentation:** Updated documentation URLs to the new domain.\n * **Documentation:** Added SERP API project example.\n * **Documentation:** Added clarifying comments for CSS selector behavior.\n * **Documentation:** Add Code of Conduct for the project (#410)\n\n\n\n## Breaking Changes Summary\n\n * **Dispatcher:** The `MemoryAdaptiveDispatcher` is now the default for `arun_many()`, changing concurrency behavior. The return type of `arun_many` depends on the `stream` parameter.\n * **Deep Crawling:** `max_depth` is now part of `CrawlerRunConfig` and controls crawl depth. Import paths for deep crawling strategies have changed.\n * **Browser Context:** The `BrowserContext` API has been updated.\n * **Models:** Many fields in data models are now optional, with default values.\n * **Scraping Mode:** `ScrapingMode` enum replaced by strategy pattern (`WebScrapingStrategy`, `LXMLWebScrapingStrategy`).\n * **Content Filter:** Removed `content_filter` parameter from `CrawlerRunConfig`. Use extraction strategies or markdown generators with filters instead.\n * **Removed:** Synchronous `WebCrawler`, CLI, and docs management functionality.\n * **Docker:** Significant changes to Docker deployment, including new requirements and configuration.\n * **File Removed** : Removed ssl_certificate.json file which might affect existing certificate validations\n * **Renamed** : final_url to redirected_url for consistency\n * **Config** : FastFilterChain has been replaced with FilterChain\n * **Deep-Crawl** : DeepCrawlStrategy.arun now returns Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]\n * **Proxy** : Removed synchronous WebCrawler support and related rate limiting configurations\n\n\n\n## Migration Guide\n\n 1. **Update Imports:** Adjust imports for `DeepCrawlStrategy`, `BreadthFirstSearchStrategy`, and related classes due to the new `deep_crawling` module structure.\n 2. **`CrawlerRunConfig`:** Move `max_depth` to `CrawlerRunConfig`. If using `content_filter`, migrate to an extraction strategy or a markdown generator with a filter.\n 3. **`arun_many()`:** Adapt code to the new `MemoryAdaptiveDispatcher` behavior and the return type.\n 4. **`BrowserContext`:** Update code using the `BrowserContext` API.\n 5. **Models:** Handle potential `None` values for optional fields in data models.\n 6. **Scraping:** Replace `ScrapingMode` enum with `WebScrapingStrategy` or `LXMLWebScrapingStrategy`.\n 7. **Docker:** Review the updated Docker documentation and adjust your deployment accordingly.\n 8. **CLI:** Migrate to the new `crwl` command and update any scripts using the old CLI.\n 9. **Proxy:** : Removed synchronous WebCrawler support and related rate limiting configurations.\n 10. **Config:** : Replace FastFilterChain to FilterChain\n\n\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../../..)\n * [ Quick Start ](../../../core/quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../../..)\n * Setup & Installation\n * [Installation](../../../core/installation/)\n * [Docker Deployment](../../../core/docker-deployment/)\n * [Quick Start](../../../core/quickstart/)\n * Blog & Changelog\n * [Blog Home](../../)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../../../core/cli/)\n * [Simple Crawling](../../../core/simple-crawling/)\n * [Deep Crawling](../../../core/deep-crawling/)\n * [Crawler Result](../../../core/crawler-result/)\n * [Browser, Crawler & LLM Config](../../../core/browser-crawler-config/)\n * [Markdown Generation](../../../core/markdown-generation/)\n * [Fit Markdown](../../../core/fit-markdown/)\n * [Page Interaction](../../../core/page-interaction/)\n * [Content Selection](../../../core/content-selection/)\n * [Cache Modes](../../../core/cache-modes/)\n * [Local Files & Raw HTML](../../../core/local-files/)\n * [Link & Media](../../../core/link-media/)\n * Advanced\n * [Overview](../../../advanced/advanced-features/)\n * [File Downloading](../../../advanced/file-downloading/)\n * [Lazy Loading](../../../advanced/lazy-loading/)\n * [Hooks & Auth](../../../advanced/hooks-auth/)\n * [Proxy & Security](../../../advanced/proxy-security/)\n * [Session Management](../../../advanced/session-management/)\n * [Multi-URL Crawling](../../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../../extraction/llm-strategies/)\n * [Clustering Strategies](../../../extraction/clustring-strategies/)\n * [Chunking](../../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../../api/async-webcrawler/)\n * [arun()](../../../api/arun/)\n * [arun_many()](../../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../../api/parameters/)\n * [CrawlResult](../../../api/crawl-result/)\n * [Strategies](../../../api/strategies/)\n\n\n\n * [Crawl4AI 0.4.3: Major Performance Boost & LLM Integration](#crawl4ai-043-major-performance-boost-llm-integration)\n * [⚡ Speed & Efficiency Improvements](#speed-efficiency-improvements)\n * [🤖 LLM Integration](#llm-integration)\n * [🔧 Core Improvements](#core-improvements)\n * [Performance Impact](#performance-impact)\n * [Getting Started](#getting-started)\n * [Stay Connected](#stay-connected)\n\n\n\n# Crawl4AI 0.4.3: Major Performance Boost & LLM Integration\n\nWe're excited to announce Crawl4AI 0.4.3, focusing on three key areas: Speed & Efficiency, LLM Integration, and Core Platform Improvements. This release significantly improves crawling performance while adding powerful new LLM-powered features.\n\n## ⚡ Speed & Efficiency Improvements\n\n### 1. Memory-Adaptive Dispatcher System\n\nThe new dispatcher system provides intelligent resource management and real-time monitoring:\n\n```\n`from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, DisplayMode from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher, CrawlerMonitor async def main(): urls = [\"https://example1.com\", \"https://example2.com\"] * 50 # Configure memory-aware dispatch dispatcher = MemoryAdaptiveDispatcher( memory_threshold_percent=80.0, # Auto-throttle at 80% memory check_interval=0.5, # Check every 0.5 seconds max_session_permit=20, # Max concurrent sessions monitor=CrawlerMonitor( # Real-time monitoring display_mode=DisplayMode.DETAILED ) ) async with AsyncWebCrawler() as crawler: results = await dispatcher.run_urls( urls=urls, crawler=crawler, config=CrawlerRunConfig() ) `\n```\n\n### 2. Streaming Support\n\nProcess crawled URLs in real-time instead of waiting for all results:\n\n```\n`config = CrawlerRunConfig(stream=True) async with AsyncWebCrawler() as crawler: async for result in await crawler.arun_many(urls, config=config): print(f\"Got result for {result.url}\") # Process each result immediately `\n```\n\n### 3. LXML-Based Scraping\n\nNew LXML scraping strategy offering up to 20x faster parsing:\n\n```\n`config = CrawlerRunConfig( scraping_strategy=LXMLWebScrapingStrategy(), cache_mode=CacheMode.ENABLED ) `\n```\n\n## 🤖 LLM Integration\n\n### 1. LLM-Powered Markdown Generation\n\nSmart content filtering and organization using LLMs:\n\n```\n`config = CrawlerRunConfig( markdown_generator=DefaultMarkdownGenerator( content_filter=LLMContentFilter( provider=\"openai/gpt-4o\", instruction=\"Extract technical documentation and code examples\" ) ) ) `\n```\n\n### 2. Automatic Schema Generation\n\nGenerate extraction schemas instantly using LLMs instead of manual CSS/XPath writing:\n\n```\n`schema = JsonCssExtractionStrategy.generate_schema( html_content, schema_type=\"CSS\", query=\"Extract product name, price, and description\" ) `\n```\n\n## 🔧 Core Improvements\n\n### 1. Proxy Support & Rotation\n\nIntegrated proxy support with automatic rotation and verification:\n\n```\n`config = CrawlerRunConfig( proxy_config={ \"server\": \"http://proxy:8080\", \"username\": \"user\", \"password\": \"pass\" } ) `\n```\n\n### 2. Robots.txt Compliance\n\nBuilt-in robots.txt support with SQLite caching:\n\n```\n`config = CrawlerRunConfig(check_robots_txt=True) result = await crawler.arun(url, config=config) if result.status_code == 403: print(\"Access blocked by robots.txt\") `\n```\n\n### 3. URL Redirection Tracking\n\nTrack final URLs after redirects:\n\n```\n`result = await crawler.arun(url) print(f\"Initial URL: {url}\") print(f\"Final URL: {result.redirected_url}\") `\n```\n\n## Performance Impact\n\n * Memory usage reduced by up to 40% with adaptive dispatcher\n * Parsing speed increased up to 20x with LXML strategy\n * Streaming reduces memory footprint for large crawls by ~60%\n\n\n\n## Getting Started\n\n```\n`pip install -U crawl4ai `\n```\n\nFor complete examples, check our [demo repository](https://github.com/unclecode/crawl4ai/examples).\n\n## Stay Connected\n\n * Star us on [GitHub](https://github.com/unclecode/crawl4ai)\n * Follow [@unclecode](https://twitter.com/unclecode)\n * Join our [Discord](https://discord.gg/crawl4ai)\n\n\n\nHappy crawling! 🕷️\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../installation/)\n * [Docker Deployment](../docker-deployment/)\n * [Quick Start](../quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../cli/)\n * [Simple Crawling](../simple-crawling/)\n * [Deep Crawling](../deep-crawling/)\n * [Crawler Result](../crawler-result/)\n * Browser, Crawler & LLM Config\n * [Markdown Generation](../markdown-generation/)\n * [Fit Markdown](../fit-markdown/)\n * [Page Interaction](../page-interaction/)\n * [Content Selection](../content-selection/)\n * [Cache Modes](../cache-modes/)\n * [Local Files & Raw HTML](../local-files/)\n * [Link & Media](../link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Browser, Crawler & LLM Configuration (Quick Overview)](#browser-crawler-llm-configuration-quick-overview)\n * [1. BrowserConfig Essentials](#1-browserconfig-essentials)\n * [2. CrawlerRunConfig Essentials](#2-crawlerrunconfig-essentials)\n * [3. LLMConfig Essentials](#3-llmconfig-essentials)\n * [4. Putting It All Together](#4-putting-it-all-together)\n * [5. Next Steps](#5-next-steps)\n * [6. Conclusion](#6-conclusion)\n\n\n\n# Browser, Crawler & LLM Configuration (Quick Overview)\n\nCrawl4AI’s flexibility stems from two key classes:\n\n1. **`BrowserConfig`** – Dictates **how** the browser is launched and behaves (e.g., headless or visible, proxy, user agent). 2. **`CrawlerRunConfig`** – Dictates **how** each **crawl** operates (e.g., caching, extraction, timeouts, JavaScript code to run, etc.). 3. **`LLMConfig`** - Dictates **how** LLM providers are configured. (model, api token, base url, temperature etc.)\n\nIn most examples, you create **one** `BrowserConfig` for the entire crawler session, then pass a **fresh** or re-used `CrawlerRunConfig` whenever you call `arun()`. This tutorial shows the most commonly used parameters. If you need advanced or rarely used fields, see the [Configuration Parameters](../../api/parameters/).\n\n## 1. BrowserConfig Essentials\n\n```\n`class BrowserConfig: def __init__( browser_type=\"chromium\", headless=True, proxy_config=None, viewport_width=1080, viewport_height=600, verbose=True, use_persistent_context=False, user_data_dir=None, cookies=None, headers=None, user_agent=None, text_mode=False, light_mode=False, extra_args=None, # ... other advanced parameters omitted here ): ... `\n```\n\n### Key Fields to Note\n\n1. **`browser_type`** - Options: `\"chromium\"`, `\"firefox\"`, or `\"webkit\"`. - Defaults to `\"chromium\"`. - If you need a different engine, specify it here.\n\n2. **`headless`** - `True`: Runs the browser in headless mode (invisible browser). - `False`: Runs the browser in visible mode, which helps with debugging.\n\n3. **`proxy_config`** - A dictionary with fields like: \n\n```\n`{ \"server\": \"http://proxy.example.com:8080\", \"username\": \"...\", \"password\": \"...\" } `\n```\n\n- Leave as `None` if a proxy is not required. \n\n4. **`viewport_width` & `viewport_height`**: - The initial window size. - Some sites behave differently with smaller or bigger viewports.\n\n5. **`verbose`** : - If `True`, prints extra logs. - Handy for debugging.\n\n6. **`use_persistent_context`** : - If `True`, uses a **persistent** browser profile, storing cookies/local storage across runs. - Typically also set `user_data_dir` to point to a folder.\n\n7. **`cookies`** & **`headers`** : - If you want to start with specific cookies or add universal HTTP headers, set them here. - E.g. `cookies=[{\"name\": \"session\", \"value\": \"abc123\", \"domain\": \"example.com\"}]`.\n\n8. **`user_agent`** : - Custom User-Agent string. If `None`, a default is used. - You can also set `user_agent_mode=\"random\"` for randomization (if you want to fight bot detection).\n\n9. **`text_mode`** & **`light_mode`** : - `text_mode=True` disables images, possibly speeding up text-only crawls. - `light_mode=True` turns off certain background features for performance. \n\n10. **`extra_args`** : - Additional flags for the underlying browser. - E.g. `[\"--disable-extensions\"]`.\n\n### Helper Methods\n\nBoth configuration classes provide a `clone()` method to create modified copies:\n\n```\n`# Create a base browser config base_browser = BrowserConfig( browser_type=\"chromium\", headless=True, text_mode=True ) # Create a visible browser config for debugging debug_browser = base_browser.clone( headless=False, verbose=True ) `\n```\n\n**Minimal Example** :\n\n```\n`from crawl4ai import AsyncWebCrawler, BrowserConfig browser_conf = BrowserConfig( browser_type=\"firefox\", headless=False, text_mode=True ) async with AsyncWebCrawler(config=browser_conf) as crawler: result = await crawler.arun(\"https://example.com\") print(result.markdown[:300]) `\n```\n\n## 2. CrawlerRunConfig Essentials\n\n```\n`class CrawlerRunConfig: def __init__( word_count_threshold=200, extraction_strategy=None, markdown_generator=None, cache_mode=None, js_code=None, wait_for=None, screenshot=False, pdf=False, enable_rate_limiting=False, rate_limit_config=None, memory_threshold_percent=70.0, check_interval=1.0, max_session_permit=20, display_mode=None, verbose=True, stream=False, # Enable streaming for arun_many() # ... other advanced parameters omitted ): ... `\n```\n\n### Key Fields to Note\n\n1. **`word_count_threshold`** : - The minimum word count before a block is considered. - If your site has lots of short paragraphs or items, you can lower it.\n\n2. **`extraction_strategy`** : - Where you plug in JSON-based extraction (CSS, LLM, etc.). - If `None`, no structured extraction is done (only raw/cleaned HTML + markdown).\n\n3. **`markdown_generator`** : - E.g., `DefaultMarkdownGenerator(...)`, controlling how HTML→Markdown conversion is done. - If `None`, a default approach is used.\n\n4. **`cache_mode`** : - Controls caching behavior (`ENABLED`, `BYPASS`, `DISABLED`, etc.). - If `None`, defaults to some level of caching or you can specify `CacheMode.ENABLED`.\n\n5. **`js_code`** : - A string or list of JS strings to execute. - Great for “Load More” buttons or user interactions. \n\n6. **`wait_for`** : - A CSS or JS expression to wait for before extracting content. - Common usage: `wait_for=\"css:.main-loaded\"` or `wait_for=\"js:() => window.loaded === true\"`.\n\n7. **`screenshot`** & **`pdf`** : - If `True`, captures a screenshot or PDF after the page is fully loaded. - The results go to `result.screenshot` (base64) or `result.pdf` (bytes).\n\n8. **`verbose`** : - Logs additional runtime details. - Overlaps with the browser’s verbosity if also set to `True` in `BrowserConfig`.\n\n9. **`enable_rate_limiting`** : - If `True`, enables rate limiting for batch processing. - Requires `rate_limit_config` to be set.\n\n10. **`memory_threshold_percent`** : - The memory threshold (as a percentage) to monitor. - If exceeded, the crawler will pause or slow down.\n\n11. **`check_interval`** : - The interval (in seconds) to check system resources. - Affects how often memory and CPU usage are monitored.\n\n12. **`max_session_permit`** : - The maximum number of concurrent crawl sessions. - Helps prevent overwhelming the system.\n\n13. **`display_mode`** : - The display mode for progress information (`DETAILED`, `BRIEF`, etc.). - Affects how much information is printed during the crawl.\n\n### Helper Methods\n\nThe `clone()` method is particularly useful for creating variations of your crawler configuration:\n\n```\n`# Create a base configuration base_config = CrawlerRunConfig( cache_mode=CacheMode.ENABLED, word_count_threshold=200, wait_until=\"networkidle\" ) # Create variations for different use cases stream_config = base_config.clone( stream=True, # Enable streaming mode cache_mode=CacheMode.BYPASS ) debug_config = base_config.clone( page_timeout=120000, # Longer timeout for debugging verbose=True ) `\n```\n\nThe `clone()` method: - Creates a new instance with all the same settings - Updates only the specified parameters - Leaves the original configuration unchanged - Perfect for creating variations without repeating all parameters\n\n## 3. LLMConfig Essentials\n\n### Key fields to note\n\n1. **`provider`** : - Which LLM provoder to use. - Possible values are `\"ollama/llama3\",\"groq/llama3-70b-8192\",\"groq/llama3-8b-8192\", \"openai/gpt-4o-mini\" ,\"openai/gpt-4o\",\"openai/o1-mini\",\"openai/o1-preview\",\"openai/o3-mini\",\"openai/o3-mini-high\",\"anthropic/claude-3-haiku-20240307\",\"anthropic/claude-3-opus-20240229\",\"anthropic/claude-3-sonnet-20240229\",\"anthropic/claude-3-5-sonnet-20240620\",\"gemini/gemini-pro\",\"gemini/gemini-1.5-pro\",\"gemini/gemini-2.0-flash\",\"gemini/gemini-2.0-flash-exp\",\"gemini/gemini-2.0-flash-lite-preview-02-05\",\"deepseek/deepseek-chat\"`_(default:`\"openai/gpt-4o-mini\"`)_\n\n2. **`api_token`** : - Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`\"GEMINI_API_KEY\"` will be read from environment variables - API token of LLM provider eg: `api_token = \"gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv\"` - Environment variable - use with prefix \"env:\" eg:`api_token = \"env: GROQ_API_KEY\"`\n\n3. **`base_url`** : - If your provider has a custom endpoint\n\n```\n`llm_config = LLMConfig(provider=\"openai/gpt-4o-mini\", api_token=os.getenv(\"OPENAI_API_KEY\")) `\n```\n\n## 4. Putting It All Together\n\nIn a typical scenario, you define **one** `BrowserConfig` for your crawler session, then create **one or more** `CrawlerRunConfig` & `LLMConfig` depending on each call’s needs:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig from crawl4ai.extraction_strategy import JsonCssExtractionStrategy async def main(): # 1) Browser config: headless, bigger viewport, no proxy browser_conf = BrowserConfig( headless=True, viewport_width=1280, viewport_height=720 ) # 2) Example extraction strategy schema = { \"name\": \"Articles\", \"baseSelector\": \"div.article\", \"fields\": [ {\"name\": \"title\", \"selector\": \"h2\", \"type\": \"text\"}, {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"} ] } extraction = JsonCssExtractionStrategy(schema) # 3) Example LLM content filtering gemini_config = LLMConfig( provider=\"gemini/gemini-1.5-pro\" api_token = \"env:GEMINI_API_TOKEN\" ) # Initialize LLM filter with specific instruction filter = LLMContentFilter( llm_config=gemini_config, # or your preferred provider instruction=\"\"\" Focus on extracting the core educational content. Include: - Key concepts and explanations - Important code examples - Essential technical details Exclude: - Navigation elements - Sidebars - Footer content Format the output as clean markdown with proper code blocks and headers. \"\"\", chunk_token_threshold=500, # Adjust based on your needs verbose=True ) md_generator = DefaultMarkdownGenerator( content_filter=filter, options={\"ignore_links\": True} # 4) Crawler run config: skip cache, use extraction run_conf = CrawlerRunConfig( markdown_generator=md_generator, extraction_strategy=extraction, cache_mode=CacheMode.BYPASS, ) async with AsyncWebCrawler(config=browser_conf) as crawler: # 4) Execute the crawl result = await crawler.arun(url=\"https://example.com/news\", config=run_conf) if result.success: print(\"Extracted content:\", result.extracted_content) else: print(\"Error:\", result.error_message) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n## 5. Next Steps\n\nFor a **detailed list** of available parameters (including advanced ones), see:\n\n * [BrowserConfig, CrawlerRunConfig & LLMConfig Reference](../../api/parameters/)\n\n\n\nYou can explore topics like:\n\n * **Custom Hooks & Auth** (Inject JavaScript or handle login forms). \n * **Session Management** (Re-use pages, preserve state across multiple calls). \n * **Magic Mode** or **Identity-based Crawling** (Fight bot detection by simulating user behavior). \n * **Advanced Caching** (Fine-tune read/write cache modes). \n\n\n\n## 6. Conclusion\n\n**BrowserConfig** , **CrawlerRunConfig** and **LLMConfig** give you straightforward ways to define:\n\n * **Which** browser to launch, how it should run, and any proxy or user agent needs. \n * **How** each crawl should behave—caching, timeouts, JavaScript code, extraction strategies, etc.\n * **Which** LLM provider to use, api token, temperature and base url for custom endpoints\n\n\n\nUse them together for **clear, maintainable** code, and when you need more specialized behavior, check out the advanced parameters in the [reference docs](../../api/parameters/). Happy crawling!\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../installation/)\n * [Docker Deployment](../docker-deployment/)\n * [Quick Start](../quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../cli/)\n * [Simple Crawling](../simple-crawling/)\n * [Deep Crawling](../deep-crawling/)\n * [Crawler Result](../crawler-result/)\n * [Browser, Crawler & LLM Config](../browser-crawler-config/)\n * [Markdown Generation](../markdown-generation/)\n * [Fit Markdown](../fit-markdown/)\n * [Page Interaction](../page-interaction/)\n * [Content Selection](../content-selection/)\n * Cache Modes\n * [Local Files & Raw HTML](../local-files/)\n * [Link & Media](../link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Crawl4AI Cache System and Migration Guide](#crawl4ai-cache-system-and-migration-guide)\n * [Overview](#overview)\n * [Old vs New Approach](#old-vs-new-approach)\n * [Migration Example](#migration-example)\n * [Common Migration Patterns](#common-migration-patterns)\n\n\n\n# Crawl4AI Cache System and Migration Guide\n\n## Overview\n\nStarting from version 0.5.0, Crawl4AI introduces a new caching system that replaces the old boolean flags with a more intuitive `CacheMode` enum. This change simplifies cache control and makes the behavior more predictable.\n\n## Old vs New Approach\n\n### Old Way (Deprecated)\n\nThe old system used multiple boolean flags: - `bypass_cache`: Skip cache entirely - `disable_cache`: Disable all caching - `no_cache_read`: Don't read from cache - `no_cache_write`: Don't write to cache\n\n### New Way (Recommended)\n\nThe new system uses a single `CacheMode` enum: - `CacheMode.ENABLED`: Normal caching (read/write) - `CacheMode.DISABLED`: No caching at all - `CacheMode.READ_ONLY`: Only read from cache - `CacheMode.WRITE_ONLY`: Only write to cache - `CacheMode.BYPASS`: Skip cache for this operation\n\n## Migration Example\n\n### Old Code (Deprecated)\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler async def use_proxy(): async with AsyncWebCrawler(verbose=True) as crawler: result = await crawler.arun( url=\"https://www.nbcnews.com/business\", bypass_cache=True # Old way ) print(len(result.markdown)) async def main(): await use_proxy() if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n### New Code (Recommended)\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CacheMode from crawl4ai.async_configs import CrawlerRunConfig async def use_proxy(): # Use CacheMode in CrawlerRunConfig config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) async with AsyncWebCrawler(verbose=True) as crawler: result = await crawler.arun( url=\"https://www.nbcnews.com/business\", config=config # Pass the configuration object ) print(len(result.markdown)) async def main(): await use_proxy() if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n## Common Migration Patterns\n\nOld Flag | New Mode \n---|--- \n`bypass_cache=True` | `cache_mode=CacheMode.BYPASS` \n`disable_cache=True` | `cache_mode=CacheMode.DISABLED` \n`no_cache_read=True` | `cache_mode=CacheMode.WRITE_ONLY` \n`no_cache_write=True` | `cache_mode=CacheMode.READ_ONLY` \n \nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../installation/)\n * [Docker Deployment](../docker-deployment/)\n * [Quick Start](../quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * Command Line Interface\n * [Simple Crawling](../simple-crawling/)\n * [Deep Crawling](../deep-crawling/)\n * [Crawler Result](../crawler-result/)\n * [Browser, Crawler & LLM Config](../browser-crawler-config/)\n * [Markdown Generation](../markdown-generation/)\n * [Fit Markdown](../fit-markdown/)\n * [Page Interaction](../page-interaction/)\n * [Content Selection](../content-selection/)\n * [Cache Modes](../cache-modes/)\n * [Local Files & Raw HTML](../local-files/)\n * [Link & Media](../link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Crawl4AI CLI Guide](#crawl4ai-cli-guide)\n * [Table of Contents](#table-of-contents)\n * [Basic Usage](#basic-usage)\n * [Quick Example of Advanced Usage](#quick-example-of-advanced-usage)\n * [Configuration](#configuration)\n * [Advanced Features](#advanced-features)\n * [Output Formats](#output-formats)\n * [Complete Examples](#complete-examples)\n * [Best Practices & Tips](#best-practices-tips)\n * [Recap](#recap)\n\n\n\n# Crawl4AI CLI Guide\n\n## Table of Contents\n\n * [Installation](#installation)\n * [Basic Usage](#basic-usage)\n * [Configuration](#configuration)\n * [Browser Configuration](#browser-configuration)\n * [Crawler Configuration](#crawler-configuration)\n * [Extraction Configuration](#extraction-configuration)\n * [Content Filtering](#content-filtering)\n * [Advanced Features](#advanced-features)\n * [LLM Q&A](#llm-qa)\n * [Structured Data Extraction](#structured-data-extraction)\n * [Content Filtering](#content-filtering-1)\n * [Output Formats](#output-formats)\n * [Examples](#examples)\n * [Configuration Reference](#configuration-reference)\n * [Best Practices & Tips](#best-practices--tips)\n\n\n\n## Basic Usage\n\nThe Crawl4AI CLI (`crwl`) provides a simple interface to the Crawl4AI library:\n\n```\n`# Basic crawling crwl https://example.com # Get markdown output crwl https://example.com -o markdown # Verbose JSON output with cache bypass crwl https://example.com -o json -v --bypass-cache # See usage examples crwl --example `\n```\n\n## Quick Example of Advanced Usage\n\nIf you clone the repository and run the following command, you will receive the content of the page in JSON format according to a JSON-CSS schema:\n\n```\n`crwl \"https://www.infoq.com/ai-ml-data-eng/\" -e docs/examples/cli/extract_css.yml -s docs/examples/cli/css_schema.json -o json; `\n```\n\n## Configuration\n\n### Browser Configuration\n\nBrowser settings can be configured via YAML file or command line parameters:\n\n```\n`# browser.yml headless: true viewport_width: 1280 user_agent_mode: \"random\" verbose: true ignore_https_errors: true `\n```\n\n```\n`# Using config file crwl https://example.com -B browser.yml # Using direct parameters crwl https://example.com -b \"headless=true,viewport_width=1280,user_agent_mode=random\" `\n```\n\n### Crawler Configuration\n\nControl crawling behavior:\n\n```\n`# crawler.yml cache_mode: \"bypass\" wait_until: \"networkidle\" page_timeout: 30000 delay_before_return_html: 0.5 word_count_threshold: 100 scan_full_page: true scroll_delay: 0.3 process_iframes: false remove_overlay_elements: true magic: true verbose: true `\n```\n\n```\n`# Using config file crwl https://example.com -C crawler.yml # Using direct parameters crwl https://example.com -c \"css_selector=#main,delay_before_return_html=2,scan_full_page=true\" `\n```\n\n### Extraction Configuration\n\nTwo types of extraction are supported:\n\n 1. CSS/XPath-based extraction: \n\n```\n`# extract_css.yml type: \"json-css\" params: verbose: true `\n```\n\n\n\n\n```\n`// css_schema.json { \"name\": \"ArticleExtractor\", \"baseSelector\": \".article\", \"fields\": [ { \"name\": \"title\", \"selector\": \"h1.title\", \"type\": \"text\" }, { \"name\": \"link\", \"selector\": \"a.read-more\", \"type\": \"attribute\", \"attribute\": \"href\" } ] } `\n```\n\n 1. LLM-based extraction: \n\n```\n`# extract_llm.yml type: \"llm\" provider: \"openai/gpt-4\" instruction: \"Extract all articles with their titles and links\" api_token: \"your-token\" params: temperature: 0.3 max_tokens: 1000 `\n```\n\n\n\n\n```\n`// llm_schema.json { \"title\": \"Article\", \"type\": \"object\", \"properties\": { \"title\": { \"type\": \"string\", \"description\": \"The title of the article\" }, \"link\": { \"type\": \"string\", \"description\": \"URL to the full article\" } } } `\n```\n\n## Advanced Features\n\n### LLM Q&A\n\nAsk questions about crawled content:\n\n```\n`# Simple question crwl https://example.com -q \"What is the main topic discussed?\" # View content then ask questions crwl https://example.com -o markdown # See content first crwl https://example.com -q \"Summarize the key points\" crwl https://example.com -q \"What are the conclusions?\" # Combined with advanced crawling crwl https://example.com \\ -B browser.yml \\ -c \"css_selector=article,scan_full_page=true\" \\ -q \"What are the pros and cons mentioned?\" `\n```\n\nFirst-time setup: - Prompts for LLM provider and API token - Saves configuration in `~/.crawl4ai/global.yml` - Supports various providers (openai/gpt-4, anthropic/claude-3-sonnet, etc.) - For case of `ollama` you do not need to provide API token. - See [LiteLLM Providers](https://docs.litellm.ai/docs/providers) for full list\n\n### Structured Data Extraction\n\nExtract structured data using CSS selectors:\n\n```\n`crwl https://example.com \\ -e extract_css.yml \\ -s css_schema.json \\ -o json `\n```\n\nOr using LLM-based extraction:\n\n```\n`crwl https://example.com \\ -e extract_llm.yml \\ -s llm_schema.json \\ -o json `\n```\n\n### Content Filtering\n\nFilter content for relevance:\n\n```\n`# filter_bm25.yml type: \"bm25\" query: \"target content\" threshold: 1.0 # filter_pruning.yml type: \"pruning\" query: \"focus topic\" threshold: 0.48 `\n```\n\n```\n`crwl https://example.com -f filter_bm25.yml -o markdown-fit `\n```\n\n## Output Formats\n\n * `all` - Full crawl result including metadata\n * `json` - Extracted structured data (when using extraction)\n * `markdown` / `md` - Raw markdown output\n * `markdown-fit` / `md-fit` - Filtered markdown for better readability\n\n\n\n## Complete Examples\n\n 1. Basic Extraction: \n\n```\n`crwl https://example.com \\ -B browser.yml \\ -C crawler.yml \\ -o json `\n```\n\n 2. Structured Data Extraction: \n\n```\n`crwl https://example.com \\ -e extract_css.yml \\ -s css_schema.json \\ -o json \\ -v `\n```\n\n 3. LLM Extraction with Filtering: \n\n```\n`crwl https://example.com \\ -B browser.yml \\ -e extract_llm.yml \\ -s llm_schema.json \\ -f filter_bm25.yml \\ -o json `\n```\n\n 4. Interactive Q&A: \n\n```\n`# First crawl and view crwl https://example.com -o markdown # Then ask questions crwl https://example.com -q \"What are the main points?\" crwl https://example.com -q \"Summarize the conclusions\" `\n```\n\n\n\n\n## Best Practices & Tips\n\n 1. **Configuration Management** :\n 2. Keep common configurations in YAML files\n 3. Use CLI parameters for quick overrides\n 4. Store sensitive data (API tokens) in `~/.crawl4ai/global.yml`\n\n 5. **Performance Optimization** :\n\n 6. Use `--bypass-cache` for fresh content\n 7. Enable `scan_full_page` for infinite scroll pages\n 8. Adjust `delay_before_return_html` for dynamic content\n\n 9. **Content Extraction** :\n\n 10. Use CSS extraction for structured content\n 11. Use LLM extraction for unstructured content\n 12. Combine with filters for focused results\n\n 13. **Q &A Workflow**:\n\n 14. View content first with `-o markdown`\n 15. Ask specific questions\n 16. Use broader context with appropriate selectors\n\n\n\n## Recap\n\nThe Crawl4AI CLI provides: - Flexible configuration via files and parameters - Multiple extraction strategies (CSS, XPath, LLM) - Content filtering and optimization - Interactive Q&A capabilities - Various output formats\n\nSite built with [MkDocs](http://www.mkdocs.org) and [Terminal for MkDocs](https://github.com/ntno/mkdocs-terminal). \n\n##### Search\n\nxClose\n\nType to start searching\n\n\n[Crawl4AI Documentation (v0.5.x)](https://docs.crawl4ai.com/)\n\n * [ Home ](../..)\n * [ Quick Start ](../quickstart/)\n * [ Search ](#)\n\n\n\n * [Home](../..)\n * Setup & Installation\n * [Installation](../installation/)\n * [Docker Deployment](../docker-deployment/)\n * [Quick Start](../quickstart/)\n * Blog & Changelog\n * [Blog Home](../../blog/)\n * [Changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md)\n * Core\n * [Command Line Interface](../cli/)\n * [Simple Crawling](../simple-crawling/)\n * [Deep Crawling](../deep-crawling/)\n * [Crawler Result](../crawler-result/)\n * [Browser, Crawler & LLM Config](../browser-crawler-config/)\n * [Markdown Generation](../markdown-generation/)\n * [Fit Markdown](../fit-markdown/)\n * [Page Interaction](../page-interaction/)\n * Content Selection\n * [Cache Modes](../cache-modes/)\n * [Local Files & Raw HTML](../local-files/)\n * [Link & Media](../link-media/)\n * Advanced\n * [Overview](../../advanced/advanced-features/)\n * [File Downloading](../../advanced/file-downloading/)\n * [Lazy Loading](../../advanced/lazy-loading/)\n * [Hooks & Auth](../../advanced/hooks-auth/)\n * [Proxy & Security](../../advanced/proxy-security/)\n * [Session Management](../../advanced/session-management/)\n * [Multi-URL Crawling](../../advanced/multi-url-crawling/)\n * [Crawl Dispatcher](../../advanced/crawl-dispatcher/)\n * [Identity Based Crawling](../../advanced/identity-based-crawling/)\n * [SSL Certificate](../../advanced/ssl-certificate/)\n * Extraction\n * [LLM-Free Strategies](../../extraction/no-llm-strategies/)\n * [LLM Strategies](../../extraction/llm-strategies/)\n * [Clustering Strategies](../../extraction/clustring-strategies/)\n * [Chunking](../../extraction/chunking/)\n * API Reference\n * [AsyncWebCrawler](../../api/async-webcrawler/)\n * [arun()](../../api/arun/)\n * [arun_many()](../../api/arun_many/)\n * [Browser, Crawler & LLM Config](../../api/parameters/)\n * [CrawlResult](../../api/crawl-result/)\n * [Strategies](../../api/strategies/)\n\n\n\n * [Content Selection](#content-selection)\n * [1. CSS-Based Selection](#1-css-based-selection)\n * [2. Content Filtering & Exclusions](#2-content-filtering-exclusions)\n * [3. Handling Iframes](#3-handling-iframes)\n * [4. Structured Extraction Examples](#4-structured-extraction-examples)\n * [5. Comprehensive Example](#5-comprehensive-example)\n * [6. Scraping Modes](#6-scraping-modes)\n * [7. Combining CSS Selection Methods](#7-combining-css-selection-methods)\n * [8. Conclusion](#8-conclusion)\n\n\n\n# Content Selection\n\nCrawl4AI provides multiple ways to **select** , **filter** , and **refine** the content from your crawls. Whether you need to target a specific CSS region, exclude entire tags, filter out external links, or remove certain domains and images, **`CrawlerRunConfig`** offers a wide range of parameters.\n\nBelow, we show how to configure these parameters and combine them for precise control.\n\n## 1. CSS-Based Selection\n\nThere are two ways to select content from a page: using `css_selector` or the more flexible `target_elements`.\n\n### 1.1 Using `css_selector`\n\nA straightforward way to **limit** your crawl results to a certain region of the page is **`css_selector`** in **`CrawlerRunConfig`** :\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig async def main(): config = CrawlerRunConfig( # e.g., first 30 items from Hacker News css_selector=\".athing:nth-child(-n+30)\" ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://news.ycombinator.com/newest\", config=config ) print(\"Partial HTML length:\", len(result.cleaned_html)) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Result** : Only elements matching that selector remain in `result.cleaned_html`.\n\n### 1.2 Using `target_elements`\n\nThe `target_elements` parameter provides more flexibility by allowing you to target **multiple elements** for content extraction while preserving the entire page context for other features:\n\n```\n`import asyncio from crawl4ai import AsyncWebCrawler, CrawlerRunConfig async def main(): config = CrawlerRunConfig( # Target article body and sidebar, but not other content target_elements=[\"article.main-content\", \"aside.sidebar\"] ) async with AsyncWebCrawler() as crawler: result = await crawler.arun( url=\"https://example.com/blog-post\", config=config ) print(\"Markdown focused on target elements\") print(\"Links from entire page still available:\", len(result.links.get(\"internal\", []))) if __name__ == \"__main__\": asyncio.run(main()) `\n```\n\n**Key difference** : With `target_elements`, the markdown generation and structural data extraction focus on those elements, but other page elements (like links, images, and tables) are still extracted from the entire page. This gives you fine-grained control over what appears in your markdown content while preserving full page context for link analysis and media collection.\n\n## 2. Content Filtering & Exclusions\n\n### 2.1 Basic Overview\n\n```\n`config = CrawlerRunConfig( # Content thresholds word_count_threshold=10, # Minimum words per block # Tag exclusions excluded_tags=['form', 'header', 'footer', 'nav'], # Link filtering exclude_external_links=True, exclude_social_media_links=True, # Block entire domains exclude_domains=[\"adtrackers.com\", \"spammynews.org\"], exclude_social_media_domains=[\"facebook.com\", \"twitter.com\"], # Media filtering exclude_external_images=True ) `\n```\n\n**Explanation** :\n\n * **`word_count_threshold`** : Ignores text blocks under X words. Helps skip trivial blocks like short nav or disclaimers. \n * **`excluded_tags`** : Removes entire tags (``, `
`, `